Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends.
- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)
Expand Down
26 changes: 19 additions & 7 deletions lib/result/ArrowResultConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,20 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string {
return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`;
}

function decimalToNumber(value: Uint32Array & Parameters<typeof bigNumToBigInt>[0]): number {
if (value.length === 4) {
const high = value[1] | 0;
const signExtension = high < 0 ? 0xffffffff : 0;
if (value[2] === signExtension && value[3] === signExtension) {
const unscaled = high * 0x100000000 + value[0];
if (Number.isSafeInteger(unscaled)) {
return unscaled;
}
}
}
return Number(bigNumToBigInt(value));
}

export default class ArrowResultConverter implements IResultsProvider<Array<any>> {
private readonly context: IClientContext;

Expand Down Expand Up @@ -407,7 +421,7 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
if (DataType.isDecimal(valueType)) {
return this.preserveBigNumericPrecision
? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
: (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale;
: (value) => decimalToNumber(value) / 10 ** valueType.scale;
}
if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) {
return (value) => Buffer.from(value);
Expand Down Expand Up @@ -504,18 +518,16 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
// Convert big number values to BigInt
// Decimals are also represented as big numbers in Arrow, so additionally process them (convert to float)
if (value instanceof Object && value[isArrowBigNumSymbol]) {
const result = bigNumToBigInt(value);
if (DataType.isDecimal(valueType)) {
// Preserve full precision as an exact string when requested (kernel);
// otherwise keep the historical lossy `number` form.
if (this.preserveBigNumericPrecision) {
return bigNumDecimalToString(result, valueType.scale);
}
return Number(result) / 10 ** valueType.scale;
return this.preserveBigNumericPrecision
? bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
: decimalToNumber(value) / 10 ** valueType.scale;
}
// A rewritten Duration Int64 surfaces as a raw `bigint`, not a BigNum
// wrapper, so it is handled in the bigint branch below — not here.
return result;
return bigNumToBigInt(value);
}

// Convert binary data to Buffer.
Expand Down
103 changes: 103 additions & 0 deletions tests/unit/result/ArrowRowMaterialization.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,109 @@ function decimal(value: string): Uint32Array {

describe('Arrow row materialization', () => {
for (const rowMode of ['object', 'array'] as const) {
it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => {
const unscaled = [
'0',
'1',
'-1',
'4294967295',
'-4294967295',
'4294967296',
'-4294967296',
'9007199254740991',
'-9007199254740991',
'9007199254740992',
'-9007199254740992',
'9007199254740993',
'-9007199254740993',
'18446744073709551615',
'-18446744073709551615',
'18446744073709551616',
'-18446744073709551616',
'79228162514264337593543950336',
'-79228162514264337593543950336',
'99999999999999999999999999999999999999',
'-99999999999999999999999999999999999999',
null,
];
const columns: Column[] = [-2, 0, 2, 18, 38].map((scale) => ({
field: new arrow.Field(`decimal${scale}`, new arrow.Decimal(scale, 38, 128)),
thriftType: TTypeId.DECIMAL_TYPE,
values: unscaled.map((value) => (value === null ? null : decimal(value))),
expected: unscaled.map((value) => (value === null ? null : Number(value) / 10 ** scale)),
}));
const converter = converterFor(columns, rowMode);
const expected = unscaled.map((value, rowIndex) =>
rowMode === 'array'
? columns.map((column) => column.expected[rowIndex])
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
);
deepStrictEqual(await converter.fetchNext({ limit: 100 }), expected);
const batch = arrow.tableFromIPC(ipc(columns)).batches[0];
deepStrictEqual((converter as any).getRows(batch.slice(3, 15), 8), expected.slice(3, 11));
const legacyRows = (converter as any).getObjectRows(batch.schema, batch.toArray());
deepStrictEqual(
rowMode === 'array'
? legacyRows.map((row: any) => columns.map((column) => row[column.field.name]))
: legacyRows,
expected,
);
});

it(`preserves nested numeric decimals and nulls (${rowMode})`, async () => {
const amount = new arrow.Decimal(2, 38, 128);
const array = new arrow.List(new arrow.Field('element', amount));
const columns: Column[] = [
{
field: new arrow.Field('array', array),
thriftType: TTypeId.ARRAY_TYPE,
values: [[decimal('-4294967297'), null, decimal('9007199254740993')], null, []],
expected: [[-42949672.97, null, Number('9007199254740993') / 100], null, []],
},
{
field: new arrow.Field(
'struct',
new arrow.Struct([new arrow.Field('amount', amount), new arrow.Field('items', array)]),
),
thriftType: TTypeId.STRUCT_TYPE,
values: [{ amount: decimal('-1'), items: [null, decimal('4294967296')] }, null, { amount: null, items: [] }],
expected: [{ amount: -0.01, items: [null, 42949672.96] }, null, { amount: null, items: [] }],
},
{
field: new arrow.Field(
'map',
new arrow.Map_(
new arrow.Field(
'entries',
new arrow.Struct<{ key: arrow.Utf8; value: arrow.Decimal }>([
new arrow.Field('key', new arrow.Utf8()),
new arrow.Field('value', amount),
]),
),
),
),
thriftType: TTypeId.MAP_TYPE,
values: [
new Map([
['negative', decimal('-9007199254740991')],
['null', null],
]),
null,
new Map(),
],
expected: [{ negative: Number('-9007199254740991') / 100, null: null }, null, {}],
},
];
deepStrictEqual(
await converterFor(columns, rowMode).fetchNext({ limit: 100 }),
[0, 1, 2].map((rowIndex) =>
rowMode === 'array'
? columns.map((column) => column.expected[rowIndex])
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
),
);
});

for (const preservePrecision of [false, true]) {
it(`preserves scalar values, types and nulls (${rowMode}, precision=${preservePrecision})`, async () => {
const timestamp = new Date('2026-01-02T03:04:05.000Z');
Expand Down
Loading