Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)
- Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open.
Expand Down
110 changes: 72 additions & 38 deletions lib/result/ArrowResultConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ const { isArrowBigNumSymbol, bigNumToBigInt } = arrowUtils;

type ArrowSchema = Schema<TypeMap>;
type ArrowSchemaField = Field<DataType<Type, TypeMap>>;
type ValueConverter = (value: any) => any;

const identity: ValueConverter = (value) => value;

/**
* Metadata key carrying the original Arrow `Duration` time unit on fields
Expand Down Expand Up @@ -284,11 +287,8 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
let result: Array<any>;
if (this.disableRowMaterialization) {
result = new Array(batchRows).fill(null);
} else if (this.rowMode === 'array') {
result = this.getRowArrays(batch, batchRows);
} else {
const table = new Table(batch);
result = this.getRows(table.schema, table.toArray().slice(0, this.remainingRows));
result = this.getRows(batch, batchRows);
}

// Reduce remaining rows count by a count of rows we just processed.
Expand Down Expand Up @@ -349,7 +349,7 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
}
}

private getRows(schema: ArrowSchema, rows: Array<StructRow | MapRow>): Array<any> {
private getObjectRows(schema: ArrowSchema, rows: Array<StructRow | MapRow>): Array<any> {
return rows.map((row) => {
// First, convert native Arrow values to corresponding plain JS objects
const record = this.convertArrowTypes(row, undefined, schema.fields, undefined);
Expand All @@ -358,29 +358,79 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
});
}

// Reads cells by column index, so same-named columns are all kept. Fields come from the
// batch data type: `RecordBatch.schema` merges same-named fields into one type.
private getRowArrays(batch: RecordBatch<TypeMap>, rowCount: number): Array<Array<any>> {
const fields: Array<ArrowSchemaField> = batch.data.type.children;
private getRows(batch: RecordBatch<TypeMap>, rowCount: number): Array<any> {
const arrayRows = this.rowMode === 'array';
const fields: Array<ArrowSchemaField> = arrayRows ? batch.data.type.children : batch.schema.fields;
const names = this.schema.map((column) => column.columnName);
if (
!arrayRows &&
(new Set(names).size !== names.length ||
names.includes('__proto__') ||
fields.length !== names.length ||
fields.some((field, index) => field.name !== names[index]))
) {
const table = new Table(batch);
return this.getObjectRows(table.schema, table.toArray().slice(0, rowCount));
}
const vectors = this.schema.map((column, index) => batch.getChildAt(index));
const rows: Array<Array<any>> = [];
for (let i = 0; i < rowCount; i += 1) {
const row: Array<any> = [];
for (let j = 0; j < this.schema.length; j += 1) {
const field = fields[j];
const value = this.convertArrowTypes(
vectors[j]?.get(i) ?? null,
field?.type,
field?.type.children || [],
field,
);
row.push(this.convertThriftColumnValue(this.schema[j], value));
const keys = arrayRows ? names.map((name, index) => index) : names;
const converters = this.schema.map((column, index) => {
const convert = this.getArrowValueConverter(fields[index]);
const descriptor = column.typeDesc.types[0]?.primitiveEntry;
const thriftType =
this.preserveBigNumericPrecision &&
(descriptor?.type === TTypeId.DECIMAL_TYPE || descriptor?.type === TTypeId.BIGINT_TYPE)
? undefined
: descriptor;
return (value: any) => (value === null ? null : convertThriftValue(thriftType, convert(value)));
});
const rows: Array<any> = new Array(rowCount);
for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
const row: any = arrayRows ? [] : {};
for (let columnIndex = 0; columnIndex < names.length; columnIndex += 1) {
const value = converters[columnIndex](vectors[columnIndex]?.get(rowIndex) ?? null);
row[keys[columnIndex]] = value;
}
rows.push(row);
rows[rowIndex] = row;
}
return rows;
}

private getArrowValueConverter(field?: ArrowSchemaField): ValueConverter {
const valueType = field?.type;
if (DataType.isTimestamp(valueType)) {
return (value) => new Date(value);
}
if (DataType.isInterval(valueType)) {
return (value) => formatArrowInterval(value, valueType);
}
if (DataType.isDecimal(valueType)) {
return this.preserveBigNumericPrecision
? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
: (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale;
}
if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) {
return (value) => Buffer.from(value);
}
if (DataType.isInt(valueType) && valueType.bitWidth === 64) {
const durationUnit = field?.metadata.get(DURATION_UNIT_METADATA_KEY);
if (durationUnit) {
return (value) => formatDurationToIntervalDayTime(value, durationUnit);
}
return this.preserveBigNumericPrecision ? identity : Number;
}
if (
DataType.isBool(valueType) ||
DataType.isInt(valueType) ||
DataType.isFloat(valueType) ||
DataType.isUtf8(valueType) ||
DataType.isDate(valueType)
) {
return identity;
}
return (value) => this.convertArrowTypes(value, valueType, valueType?.children || [], field);
}

private convertArrowTypes(
value: any,
valueType: DataType | undefined,
Expand Down Expand Up @@ -525,20 +575,4 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>

return result;
}

// Per-cell equivalent of `convertThriftTypes`, used by array rows. Kept separate so
// the default object path stays as is.
private convertThriftColumnValue(column: TColumnDesc, value: any): any {
if (value === null) {
return null;
}
const typeDescriptor = column.typeDesc.types[0]?.primitiveEntry;
if (
this.preserveBigNumericPrecision &&
(typeDescriptor?.type === TTypeId.DECIMAL_TYPE || typeDescriptor?.type === TTypeId.BIGINT_TYPE)
) {
return value;
}
return convertThriftValue(typeDescriptor, value);
}
}
93 changes: 93 additions & 0 deletions tests/e2e/row-materialization.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import { expect } from 'chai';
import { DBSQLClient } from '../../lib';
import { ConnectionOptions } from '../../lib/contracts/IDBSQLClient';
import { InternalConnectionOptions } from '../../lib/contracts/InternalConnectionOptions';
import config from './utils/config';

const useKernel = process.env.E2E_USE_KERNEL === '1';
const largeInteger = BigInt('9007199254740993');

describe(`Arrow row materialization (${useKernel ? 'kernel' : 'Thrift'})`, () => {
for (const rowMode of ['object', 'array'] as const) {
for (const useArrowNativeTypes of [false, true]) {
for (const preserveBigNumericPrecision of [false, true]) {
it(`preserves values (${rowMode}, native=${useArrowNativeTypes}, precision=${preserveBigNumericPrecision})`, async () => {
const client = new DBSQLClient();
const base = client.getConfig();
client.getConfig = () => ({ ...base, useArrowNativeTypes });
try {
await client.connect({
host: config.host,
path: config.path,
token: config.token,
useKernel,
preserveBigNumericPrecision,
} as ConnectionOptions & InternalConnectionOptions);
const session = await client.openSession();
try {
const operation = await session.executeStatement(
`
SELECT CAST(id AS INT) AS id,
IF(id = 1, NULL, true) AS b,
IF(id = 1, NULL, CAST(-7 AS TINYINT)) AS tiny,
IF(id = 1, NULL, CAST(300 AS SMALLINT)) AS small,
IF(id = 1, NULL, CAST(9007199254740993 AS BIGINT)) AS big,
IF(id = 1, NULL, CAST(1.25 AS DECIMAL(30,4))) AS dec,
IF(id = 1, NULL, CAST(1.25 AS FLOAT)) AS flt,
IF(id = 1, NULL, CAST(-2.5 AS DOUBLE)) AS dbl,
IF(id = 1, NULL, 'hello🌍') AS txt,
IF(id = 1, NULL, unhex('0080ff')) AS bin,
IF(id = 1, NULL, DATE '2026-01-02') AS dt,
IF(id = 1, NULL, TIMESTAMP '2026-01-02 03:04:05') AS ts,
IF(id = 1, NULL, array(1, NULL, 3)) AS arr,
IF(id = 1, NULL, map('a', 1, 'b', NULL)) AS mp,
IF(id = 1, NULL, named_struct('items', array(1, NULL), 'text', 'x')) AS st,
IF(id = 1, NULL, INTERVAL '1-2' YEAR TO MONTH) AS ym,
IF(id = 1, NULL, INTERVAL '1 02:03:04.123456' DAY TO SECOND) AS day_time
FROM range(3) ORDER BY id
`,
{ rowMode },
);
try {
const precise = preserveBigNumericPrecision;
const values = {
id: 0,
b: true,
tiny: -7,
small: 300,
big: precise ? largeInteger : 9007199254740992,
dec: precise ? '1.2500' : 1.25,
flt: 1.25,
dbl: -2.5,
txt: 'hello🌍',
bin: Buffer.from([0, 128, 255]),
dt: new Date('2026-01-02T00:00:00Z'),
ts: useKernel || useArrowNativeTypes ? new Date('2026-01-02T03:04:05Z') : '2026-01-02 03:04:05',
arr: [1, null, 3],
mp: { a: 1, b: null },
st: { items: [1, null], text: 'x' },
ym: '1-2',
day_time: '1 02:03:04.123456000',
};
const expected = [
values,
Object.fromEntries(Object.keys(values).map((key) => [key, key === 'id' ? 1 : null])),
{ ...values, id: 2 },
];
expect(await operation.fetchAll()).to.deep.equal(
rowMode === 'array' ? expected.map(Object.values) : expected,
);
} finally {
await operation.close();
}
} finally {
await session.close();
}
} finally {
await client.close();
}
});
}
}
}
});
Loading
Loading