diff --git a/CHANGELOG.md b/CHANGELOG.md index f8aeb947..68552506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index b5d558a6..22f81a56 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -192,6 +192,20 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string { return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`; } +function decimalToNumber(value: Uint32Array & Parameters[0]): number { + if (value.length === 4) { + const high = value[1] | 0; + const signExtension = high < 0 ? 0xffffffff : 0; + if (value[2] === signExtension && value[3] === signExtension) { + const unscaled = high * 0x100000000 + value[0]; + if (Number.isSafeInteger(unscaled)) { + return unscaled; + } + } + } + return Number(bigNumToBigInt(value)); +} + export default class ArrowResultConverter implements IResultsProvider> { private readonly context: IClientContext; @@ -407,7 +421,7 @@ export default class ArrowResultConverter implements IResultsProvider if (DataType.isDecimal(valueType)) { return this.preserveBigNumericPrecision ? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale) - : (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale; + : (value) => decimalToNumber(value) / 10 ** valueType.scale; } if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) { return (value) => Buffer.from(value); @@ -504,18 +518,16 @@ export default class ArrowResultConverter implements IResultsProvider // Convert big number values to BigInt // Decimals are also represented as big numbers in Arrow, so additionally process them (convert to float) if (value instanceof Object && value[isArrowBigNumSymbol]) { - const result = bigNumToBigInt(value); if (DataType.isDecimal(valueType)) { // Preserve full precision as an exact string when requested (kernel); // otherwise keep the historical lossy `number` form. - if (this.preserveBigNumericPrecision) { - return bigNumDecimalToString(result, valueType.scale); - } - return Number(result) / 10 ** valueType.scale; + return this.preserveBigNumericPrecision + ? bigNumDecimalToString(bigNumToBigInt(value), valueType.scale) + : decimalToNumber(value) / 10 ** valueType.scale; } // A rewritten Duration Int64 surfaces as a raw `bigint`, not a BigNum // wrapper, so it is handled in the bigint branch below — not here. - return result; + return bigNumToBigInt(value); } // Convert binary data to Buffer. diff --git a/tests/unit/result/ArrowRowMaterialization.test.ts b/tests/unit/result/ArrowRowMaterialization.test.ts index 3e406f57..eb006707 100644 --- a/tests/unit/result/ArrowRowMaterialization.test.ts +++ b/tests/unit/result/ArrowRowMaterialization.test.ts @@ -73,6 +73,109 @@ function decimal(value: string): Uint32Array { describe('Arrow row materialization', () => { for (const rowMode of ['object', 'array'] as const) { + it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => { + const unscaled = [ + '0', + '1', + '-1', + '4294967295', + '-4294967295', + '4294967296', + '-4294967296', + '9007199254740991', + '-9007199254740991', + '9007199254740992', + '-9007199254740992', + '9007199254740993', + '-9007199254740993', + '18446744073709551615', + '-18446744073709551615', + '18446744073709551616', + '-18446744073709551616', + '79228162514264337593543950336', + '-79228162514264337593543950336', + '99999999999999999999999999999999999999', + '-99999999999999999999999999999999999999', + null, + ]; + const columns: Column[] = [-2, 0, 2, 18, 38].map((scale) => ({ + field: new arrow.Field(`decimal${scale}`, new arrow.Decimal(scale, 38, 128)), + thriftType: TTypeId.DECIMAL_TYPE, + values: unscaled.map((value) => (value === null ? null : decimal(value))), + expected: unscaled.map((value) => (value === null ? null : Number(value) / 10 ** scale)), + })); + const converter = converterFor(columns, rowMode); + const expected = unscaled.map((value, rowIndex) => + rowMode === 'array' + ? columns.map((column) => column.expected[rowIndex]) + : Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])), + ); + deepStrictEqual(await converter.fetchNext({ limit: 100 }), expected); + const batch = arrow.tableFromIPC(ipc(columns)).batches[0]; + deepStrictEqual((converter as any).getRows(batch.slice(3, 15), 8), expected.slice(3, 11)); + const legacyRows = (converter as any).getObjectRows(batch.schema, batch.toArray()); + deepStrictEqual( + rowMode === 'array' + ? legacyRows.map((row: any) => columns.map((column) => row[column.field.name])) + : legacyRows, + expected, + ); + }); + + it(`preserves nested numeric decimals and nulls (${rowMode})`, async () => { + const amount = new arrow.Decimal(2, 38, 128); + const array = new arrow.List(new arrow.Field('element', amount)); + const columns: Column[] = [ + { + field: new arrow.Field('array', array), + thriftType: TTypeId.ARRAY_TYPE, + values: [[decimal('-4294967297'), null, decimal('9007199254740993')], null, []], + expected: [[-42949672.97, null, Number('9007199254740993') / 100], null, []], + }, + { + field: new arrow.Field( + 'struct', + new arrow.Struct([new arrow.Field('amount', amount), new arrow.Field('items', array)]), + ), + thriftType: TTypeId.STRUCT_TYPE, + values: [{ amount: decimal('-1'), items: [null, decimal('4294967296')] }, null, { amount: null, items: [] }], + expected: [{ amount: -0.01, items: [null, 42949672.96] }, null, { amount: null, items: [] }], + }, + { + field: new arrow.Field( + 'map', + new arrow.Map_( + new arrow.Field( + 'entries', + new arrow.Struct<{ key: arrow.Utf8; value: arrow.Decimal }>([ + new arrow.Field('key', new arrow.Utf8()), + new arrow.Field('value', amount), + ]), + ), + ), + ), + thriftType: TTypeId.MAP_TYPE, + values: [ + new Map([ + ['negative', decimal('-9007199254740991')], + ['null', null], + ]), + null, + new Map(), + ], + expected: [{ negative: Number('-9007199254740991') / 100, null: null }, null, {}], + }, + ]; + deepStrictEqual( + await converterFor(columns, rowMode).fetchNext({ limit: 100 }), + [0, 1, 2].map((rowIndex) => + rowMode === 'array' + ? columns.map((column) => column.expected[rowIndex]) + : Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])), + ), + ); + }); + for (const preservePrecision of [false, true]) { it(`preserves scalar values, types and nulls (${rowMode}, precision=${preservePrecision})`, async () => { const timestamp = new Date('2026-01-02T03:04:05.000Z');