From 72e2af928a01cc4b3982c5373121dc427175742f Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 09:33:51 +0000 Subject: [PATCH 1/7] fix(thrift): honor per-statement rowLimit Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + CONNECTION_PARAMETERS.md | 4 +-- lib/contracts/IDBSQLSession.ts | 5 ++-- lib/thrift-backend/ThriftSessionBackend.ts | 11 +++---- tests/unit/DBSQLSession.test.ts | 35 ++++++++++++++++++++++ 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..b3e898db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Thrift now honors the per-statement `rowLimit` server-side cap, matching the kernel backend. Previously this option was ignored on Thrift, so callers that already set it may now receive fewer rows; `maxRows` fetch sizing is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..031402d1 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -217,7 +217,7 @@ telemetry implementation, so only the options explicitly forwarded by | `namedParameters` | `Record` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. | | `ordinalParameters` | `Array` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. | | `queryTags` | `Record` | ✅ | ✅ | — | Serialized into the per-statement `query_tags` conf overlay on both. | -| `rowLimit` | `number` | ❌ | ✅ | — | Kernel-only server-side row cap. Thrift logs and ignores it. | +| `rowLimit` | `number` | ✅ | ✅ | — | Per-statement server-side row cap, separate from `maxRows` fetch sizing. Omitted by default (no driver-imposed cap). | | `statementConf` | `Record` | ❌ | ✅ | — | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty. | ## Metadata-operation request parameters @@ -249,6 +249,6 @@ volume operations. Kernel-only capabilities include connection-pool sizing, proxy bypass hosts, independent TLS hostname verification, optional U2M disk caching, and -per-statement `rowLimit`/`statementConf`. +per-statement `statementConf`. The Kernel path remains internal and unstable; its behavior may change. diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 12d7d6c2..067db4e9 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -48,9 +48,8 @@ export type ExecuteStatementOptions = { */ queryTags?: Record; /** - * kernel-only: server-side row cap for this statement (kernel `row_limit`). The - * Thrift backend has no execute-time server cap, so this is a no-op there; - * use `maxRows` for the cross-backend client-side fetch limit. + * Per-statement server-side row cap on both backends. Omitted by default (no + * driver-imposed cap). Distinct from `maxRows`, which controls fetch-page size. */ rowLimit?: number; /** diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 49ec539a..ae026d04 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -184,17 +184,13 @@ export default class ThriftSessionBackend implements ISessionBackend { const driver = await this.context.getDriver(); const clientConfig = this.context.getConfig(); - // `rowLimit` / `statementConf` are kernel-backend (kernel) options with no - // Thrift wire equivalent. Surface a debug breadcrumb rather than dropping - // them silently, so a caller that set them on the Thrift path has signal. - if (options.rowLimit !== undefined || options.statementConf !== undefined) { + if (options.statementConf !== undefined) { this.context .getLogger() .log( LogLevel.warn, - 'ThriftSessionBackend.executeStatement: rowLimit / statementConf are kernel-backend (useKernel) ' + - 'options with no Thrift wire equivalent — they are IGNORED on the Thrift path (e.g. rowLimit ' + - 'will not cap the result set). Use the kernel backend (useKernel) to honour them.', + 'ThriftSessionBackend.executeStatement: statementConf is a kernel-backend (useKernel) option ' + + 'and is IGNORED on the Thrift path. Use the kernel backend (useKernel) to honour it.', ); } @@ -202,6 +198,7 @@ export default class ThriftSessionBackend implements ISessionBackend { sessionHandle: this.sessionHandle, statement, queryTimeout: options.queryTimeout ? numberToInt64(options.queryTimeout) : undefined, + resultRowLimit: options.rowLimit !== undefined ? numberToInt64(options.rowLimit) : undefined, runAsync: true, ...getDirectResultsOptions(options.maxRows, clientConfig), ...getArrowOptions(clientConfig, this.serverProtocolVersion), diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 00498c54..83cd47c2 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -8,6 +8,7 @@ import DBSQLOperation from '../../lib/DBSQLOperation'; import { PrimaryKeysRequest } from '../../lib/contracts/IDBSQLSession'; import ISessionBackend from '../../lib/contracts/ISessionBackend'; import ParameterError from '../../lib/errors/ParameterError'; +import { LogLevel } from '../../lib/contracts/IDBSQLLogger'; import { DBSQLParameter } from '../../lib/DBSQLParameter'; import { TSessionHandle, TProtocolVersion, TOperationState, TStatusCode } from '../../thrift/TCLIService_types'; import ClientContextStub from './.stubs/ClientContextStub'; @@ -80,6 +81,40 @@ describe('DBSQLSession', () => { expect(result).instanceOf(DBSQLOperation); }); + for (const rowLimit of [7, 0, undefined]) { + it(`should ${rowLimit === undefined ? 'omit an unset' : `forward ${rowLimit} as the`} rowLimit`, async () => { + const context = new ClientContextStub(); + const driver = sinon.spy(context.driver); + const logger = sinon.spy(context.logger, 'log'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + + await session.executeStatement('SELECT * FROM table', { rowLimit, maxRows: 100 }); + + expect(driver.executeStatement.callCount).to.equal(1); + const request = driver.executeStatement.firstCall.args[0]; + if (rowLimit === undefined) { + expect(request.resultRowLimit).to.not.exist; + } else { + expect(request.resultRowLimit).to.be.instanceOf(Int64); + expect(request.resultRowLimit?.toNumber()).to.equal(rowLimit); + } + expect(request.getDirectResults?.maxRows.toNumber()).to.equal(100); + expect(logger.calledWith(LogLevel.warn)).to.be.false; + }); + } + + it('should warn only about statementConf when rowLimit is also set', async () => { + const context = new ClientContextStub(); + const logger = sinon.spy(context.logger, 'log'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + + await session.executeStatement('SELECT 1', { rowLimit: 7, statementConf: {} }); + + const warnings = logger.getCalls().filter((call) => call.args[0] === LogLevel.warn); + expect(warnings).to.have.lengthOf(1); + expect(warnings[0].args[1]).to.include('statementConf').and.not.include('rowLimit'); + }); + describe('Arrow support', () => { it('should not use Arrow if disabled in options', async () => { const session = createSessionForTest({ From f0fa993fef14d17036c6a4f0ab8bd0b25a0f1672 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 10:05:22 +0000 Subject: [PATCH 2/7] fix(thrift): align row limit coercion and document upgrade effects Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 3 +++ lib/contracts/IDBSQLSession.ts | 5 +++++ lib/kernel/KernelSessionBackend.ts | 2 +- lib/thrift-backend/ThriftSessionBackend.ts | 2 +- tests/unit/DBSQLSession.test.ts | 9 ++++++--- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e898db..c92862ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Thrift now honors the per-statement `rowLimit` server-side cap, matching the kernel backend. Previously this option was ignored on Thrift, so callers that already set it may now receive fewer rows; `maxRows` fetch sizing is unchanged. +- Thrift now honors the per-statement `rowLimit` server-side cap, matching the kernel backend. Previously ignored values now cap results: `0` or values that truncate to zero return no rows (unlike JDBC's unlimited `setMaxRows(0)`), while out-of-range integers (valid Thrift range: `0..2147483646`) and infinities fail `executeStatement`. Runtime coercion matches the kernel: numeric strings become numbers, and `null`/`NaN` produce a zero-row cap; `maxRows` fetch sizing is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 031402d1..1c141923 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -220,6 +220,9 @@ telemetry implementation, so only the options explicitly forwarded by | `rowLimit` | `number` | ✅ | ✅ | — | Per-statement server-side row cap, separate from `maxRows` fetch sizing. Omitted by default (no driver-imposed cap). | | `statementConf` | `Record` | ❌ | ✅ | — | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty. | +Thrift sends `resultRowLimit` without a protocol-version gate, as JDBC and Python do. +Servers that do not support this field ignore it. + ## Metadata-operation request parameters The public session metadata methods also accept request objects. These are not diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 067db4e9..45b82ca6 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -50,6 +50,11 @@ export type ExecuteStatementOptions = { /** * Per-statement server-side row cap on both backends. Omitted by default (no * driver-imposed cap). Distinct from `maxRows`, which controls fetch-page size. + * On Thrift, values truncate to integers in 0..2147483646; zero returns no rows + * (unlike JDBC's unlimited setMaxRows(0)). Out-of-range values and infinities + * fail executeStatement; Thrift previously ignored this option. Runtime + * coercion matches the kernel: numeric strings become numbers; null and NaN + * produce a zero-row cap, as do values that truncate to zero. */ rowLimit?: number; /** diff --git a/lib/kernel/KernelSessionBackend.ts b/lib/kernel/KernelSessionBackend.ts index dd55dd4c..3b2b949a 100644 --- a/lib/kernel/KernelSessionBackend.ts +++ b/lib/kernel/KernelSessionBackend.ts @@ -143,7 +143,7 @@ export default class KernelSessionBackend implements ISessionBackend { * - `queryTimeout` → NO-OP on kernel (SQL Warehouses use `STATEMENT_TIMEOUT`); * never forwarded to the kernel and never applied as a client-side * deadline — see the note in `executeStatement`; - * - `rowLimit` → `rowLimit` (kernel-only server-side row cap); + * - `rowLimit` → `rowLimit` (server-side row cap); * - `queryTags` → serialised into the conf overlay's reserved * `query_tags` key (the same wire shape Thrift's `serializeQueryTags` * produces), merged with any explicit `statementConf`. diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index ae026d04..8ed90683 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -198,7 +198,7 @@ export default class ThriftSessionBackend implements ISessionBackend { sessionHandle: this.sessionHandle, statement, queryTimeout: options.queryTimeout ? numberToInt64(options.queryTimeout) : undefined, - resultRowLimit: options.rowLimit !== undefined ? numberToInt64(options.rowLimit) : undefined, + resultRowLimit: options.rowLimit !== undefined ? numberToInt64(Number(options.rowLimit)) : undefined, runAsync: true, ...getDirectResultsOptions(options.maxRows, clientConfig), ...getArrowOptions(clientConfig, this.serverProtocolVersion), diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 83cd47c2..42ac1c1b 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -81,14 +81,17 @@ describe('DBSQLSession', () => { expect(result).instanceOf(DBSQLOperation); }); - for (const rowLimit of [7, 0, undefined]) { + for (const rowLimit of [7, 0, undefined, '100', null]) { it(`should ${rowLimit === undefined ? 'omit an unset' : `forward ${rowLimit} as the`} rowLimit`, async () => { const context = new ClientContextStub(); const driver = sinon.spy(context.driver); const logger = sinon.spy(context.logger, 'log'); const session = createSessionForTest({ handle: sessionHandleStub, context }); - await session.executeStatement('SELECT * FROM table', { rowLimit, maxRows: 100 }); + await session.executeStatement('SELECT * FROM table', { + rowLimit: rowLimit as number | undefined, + maxRows: 100, + }); expect(driver.executeStatement.callCount).to.equal(1); const request = driver.executeStatement.firstCall.args[0]; @@ -96,7 +99,7 @@ describe('DBSQLSession', () => { expect(request.resultRowLimit).to.not.exist; } else { expect(request.resultRowLimit).to.be.instanceOf(Int64); - expect(request.resultRowLimit?.toNumber()).to.equal(rowLimit); + expect(request.resultRowLimit?.toNumber()).to.equal(Number(rowLimit)); } expect(request.getDirectResults?.maxRows.toNumber()).to.equal(100); expect(logger.calledWith(LogLevel.warn)).to.be.false; From 36cfd70e836ec3370b6d6fc9f548406eb89bd870 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 18:24:31 +0000 Subject: [PATCH 3/7] fix(thrift): enforce total row caps and validate limits before i64 conversion Enforce rowLimit across result pages and all Thrift result formats, including legacy columnar overflow. Validate the documented normalized range before constructing Int64 so large inputs cannot wrap into a different limit. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 3 +- CONNECTION_PARAMETERS.md | 6 +- lib/contracts/IDBSQLSession.ts | 11 +-- lib/thrift-backend/ThriftOperationBackend.ts | 24 ++++++- lib/thrift-backend/ThriftSessionBackend.ts | 16 ++++- tests/e2e/row_limit.test.ts | 72 ++++++++++++++++++++ tests/unit/DBSQLSession.test.ts | 71 ++++++++++++++++++- 7 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/row_limit.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c92862ec..a141d4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- Thrift now honors the per-statement `rowLimit` server-side cap, matching the kernel backend. Previously ignored values now cap results: `0` or values that truncate to zero return no rows (unlike JDBC's unlimited `setMaxRows(0)`), while out-of-range integers (valid Thrift range: `0..2147483646`) and infinities fail `executeStatement`. Runtime coercion matches the kernel: numeric strings become numbers, and `null`/`NaN` produce a zero-row cap; `maxRows` fetch sizing is unchanged. +- Thrift now honors the previously ignored per-statement `rowLimit`, sends the server cap, and enforces the total locally across fetches and all result formats (including legacy columnar overflow). `0` returns no rows, unlike JDBC's unlimited `setMaxRows(0)`; `maxRows` remains fetch-page sizing. +- Thrift converts `rowLimit` with `Number` (`null`/`NaN` become zero) and truncates toward zero, then rejects values outside `0..2147483646`, including infinities, with `ParameterError` before sending a request. Kernel behavior is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 1c141923..485bf0fd 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -221,7 +221,11 @@ telemetry implementation, so only the options explicitly forwarded by | `statementConf` | `Record` | ❌ | ✅ | — | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty. | Thrift sends `resultRowLimit` without a protocol-version gate, as JDBC and Python do. -Servers that do not support this field ignore it. +Thrift also enforces the total cap client-side across fetches and all result formats, +including on servers that ignore this field. It applies `Number` coercion (`null`/`NaN` +become zero), truncates toward zero, then requires `0..2147483646`; out-of-range +values and infinities throw `ParameterError` before any statement request. Zero +returns no rows. Kernel behavior is unchanged. ## Metadata-operation request parameters diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 45b82ca6..d6f166aa 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -50,11 +50,12 @@ export type ExecuteStatementOptions = { /** * Per-statement server-side row cap on both backends. Omitted by default (no * driver-imposed cap). Distinct from `maxRows`, which controls fetch-page size. - * On Thrift, values truncate to integers in 0..2147483646; zero returns no rows - * (unlike JDBC's unlimited setMaxRows(0)). Out-of-range values and infinities - * fail executeStatement; Thrift previously ignored this option. Runtime - * coercion matches the kernel: numeric strings become numbers; null and NaN - * produce a zero-row cap, as do values that truncate to zero. + * Thrift also enforces the total cap locally across fetches and result formats. + * It applies Number coercion (numeric strings become numbers, null/NaN become + * zero), then truncates toward zero. The resulting integer must be in + * 0..2147483646; otherwise executeStatement throws ParameterError before sending + * the request. Zero returns no rows (unlike JDBC's unlimited setMaxRows(0)). + * Thrift previously ignored this option. Kernel behavior is unchanged. */ rowLimit?: number; /** diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 10f2d03e..05a2b3ec 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -37,6 +37,7 @@ interface ThriftOperationBackendOptions { functionCatalog?: string | null; emptyResultOnNotFound?: TTableSchema; rowMode?: RowMode; + rowLimit?: number; } async function delay(ms?: number): Promise { @@ -105,6 +106,8 @@ export default class ThriftOperationBackend implements IOperationBackend { private resultHandler?: ResultSlicer; + private remainingRows?: number; + constructor({ handle, directResults, @@ -112,12 +115,14 @@ export default class ThriftOperationBackend implements IOperationBackend { functionCatalog, emptyResultOnNotFound, rowMode, + rowLimit, }: ThriftOperationBackendOptions) { this.operationHandle = handle; this.context = context; this.functionCatalog = functionCatalog; this.emptyResultOnNotFound = emptyResultOnNotFound; this.rowMode = rowMode; + this.remainingRows = rowLimit; this.metadata = directResults?.resultSetMetadata; const useOnlyPrefetchedResults = Boolean(directResults?.closeOperation); @@ -168,6 +173,10 @@ export default class ThriftOperationBackend implements IOperationBackend { disableBuffering?: boolean; isClosed?: () => boolean; }): Promise> { + if (this.remainingRows === 0) { + return []; + } + const resultHandler = await this.getResultHandler(); // All the library code is Promise-based, however, since Promises are microtasks, @@ -194,7 +203,16 @@ export default class ThriftOperationBackend implements IOperationBackend { return []; } - const rows = await resultHandler.fetchNext({ limit, disableBuffering }); + let rows = await resultHandler.fetchNext({ + limit: this.remainingRows === undefined ? limit : Math.min(limit, this.remainingRows), + disableBuffering, + }); + if (this.remainingRows !== undefined) { + if (rows.length > this.remainingRows) { + rows = rows.slice(0, this.remainingRows); + } + this.remainingRows -= rows.length; + } if (this.functionCatalog === undefined) { return rows; } @@ -204,6 +222,10 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async hasMore(): Promise { + if (this.remainingRows === 0) { + return false; + } + const resultHandler = await this.getResultHandler(); return resultHandler.hasMore(); } diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 8ed90683..fc6523a7 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -181,6 +181,11 @@ export default class ThriftSessionBackend implements ISessionBackend { } public async executeStatement(statement: string, options: ExecuteStatementOptions): Promise { + const rowLimit = options.rowLimit === undefined ? undefined : Math.trunc(Number(options.rowLimit) || 0); + if (rowLimit !== undefined && (!Number.isFinite(rowLimit) || rowLimit < 0 || rowLimit > 2147483646)) { + throw new ParameterError('rowLimit must be between 0 and 2147483646'); + } + const driver = await this.context.getDriver(); const clientConfig = this.context.getConfig(); @@ -198,7 +203,7 @@ export default class ThriftSessionBackend implements ISessionBackend { sessionHandle: this.sessionHandle, statement, queryTimeout: options.queryTimeout ? numberToInt64(options.queryTimeout) : undefined, - resultRowLimit: options.rowLimit !== undefined ? numberToInt64(Number(options.rowLimit)) : undefined, + resultRowLimit: rowLimit !== undefined ? numberToInt64(rowLimit) : undefined, runAsync: true, ...getDirectResultsOptions(options.maxRows, clientConfig), ...getArrowOptions(clientConfig, this.serverProtocolVersion), @@ -222,7 +227,7 @@ export default class ThriftSessionBackend implements ISessionBackend { } const response = await driver.executeStatement(request); - return this.createOperationBackend(response, { rowMode: options.rowMode }); + return this.createOperationBackend(response, { rowMode: options.rowMode, rowLimit }); } public async getTypeInfo(request: TypeInfoRequest): Promise { @@ -373,7 +378,11 @@ export default class ThriftSessionBackend implements ISessionBackend { private createOperationBackend( response: OperationResponseShape, - { functionCatalog, rowMode }: { functionCatalog?: string | null; rowMode?: RowMode } = {}, + { + functionCatalog, + rowMode, + rowLimit, + }: { functionCatalog?: string | null; rowMode?: RowMode; rowLimit?: number } = {}, ): IOperationBackend { Status.assert(response.status); const handle = definedOrError(response.operationHandle); @@ -383,6 +392,7 @@ export default class ThriftSessionBackend implements ISessionBackend { context: this.context, functionCatalog, rowMode, + rowLimit, }); } } diff --git a/tests/e2e/row_limit.test.ts b/tests/e2e/row_limit.test.ts new file mode 100644 index 00000000..93c15d69 --- /dev/null +++ b/tests/e2e/row_limit.test.ts @@ -0,0 +1,72 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { after, before, describe, it } from 'mocha'; +import { DBSQLClient } from '../../lib'; +import { ConnectionOptions } from '../../lib/contracts/IDBSQLClient'; +import { InternalConnectionOptions } from '../../lib/contracts/InternalConnectionOptions'; +import JsonResultHandler from '../../lib/result/JsonResultHandler'; +import ArrowResultConverter from '../../lib/result/ArrowResultConverter'; +import config from './utils/config'; + +describe('rowLimit', () => { + for (const useKernel of [false, true]) { + for (const arrowEnabled of useKernel ? [true] : [false, true]) { + describe(`${useKernel ? 'kernel' : 'Thrift'} (arrowEnabled=${arrowEnabled})`, () => { + const client = new DBSQLClient(); + + before(async () => { + const clientConfig = client.getConfig(); + sinon.stub(client, 'getConfig').returns({ ...clientConfig, arrowEnabled }); + await client.connect({ + host: config.host, + path: config.path, + token: config.token, + useKernel, + } as ConnectionOptions & InternalConnectionOptions); + }); + + after(async () => { + await client.close(); + }); + + for (const disableBuffering of [false, true]) { + for (const rowLimit of [0, 3, 20, 25]) { + it(`caps ${rowLimit} rows across fetches (disableBuffering=${disableBuffering})`, async () => { + const session = await client.openSession(); + try { + const operation = await session.executeStatement('SELECT id FROM RANGE(20) ORDER BY id', { + rowLimit, + maxRows: disableBuffering ? null : 2, + useCloudFetch: false, + }); + try { + await operation.finished(); + if (!useKernel) { + // @ts-expect-error Inspect the real negotiated format, not just the requested config. + const handler = await operation.getResultHandler(); + expect(handler.source).to.be.instanceOf(arrowEnabled ? ArrowResultConverter : JsonResultHandler); + } + const rows: number[] = []; + // eslint-disable-next-line no-await-in-loop + while (await operation.hasMoreRows()) { + // eslint-disable-next-line no-await-in-loop + const chunk = await operation.fetchChunk({ maxRows: 2, disableBuffering }); + rows.push(...chunk.map((row: any) => Number(row.id))); + } + expect(rows).to.deep.equal(Array.from({ length: Math.min(rowLimit, 20) }, (_, index) => index)); + expect(await operation.hasMoreRows()).to.be.false; + expect(await operation.fetchChunk({ maxRows: 2, disableBuffering })).to.deep.equal([]); + expect(await operation.fetchAll()).to.deep.equal([]); + } finally { + await operation.close(); + } + } finally { + await session.close(); + } + }); + } + } + }); + } + } +}); diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 42ac1c1b..7581b292 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -81,7 +81,7 @@ describe('DBSQLSession', () => { expect(result).instanceOf(DBSQLOperation); }); - for (const rowLimit of [7, 0, undefined, '100', null]) { + for (const rowLimit of [7, 0, 2147483646, undefined, '100', null, NaN, 3.9, -0.5]) { it(`should ${rowLimit === undefined ? 'omit an unset' : `forward ${rowLimit} as the`} rowLimit`, async () => { const context = new ClientContextStub(); const driver = sinon.spy(context.driver); @@ -99,13 +99,80 @@ describe('DBSQLSession', () => { expect(request.resultRowLimit).to.not.exist; } else { expect(request.resultRowLimit).to.be.instanceOf(Int64); - expect(request.resultRowLimit?.toNumber()).to.equal(Number(rowLimit)); + expect(request.resultRowLimit?.toNumber()).to.equal(Math.trunc(Number(rowLimit) || 0)); } expect(request.getDirectResults?.maxRows.toNumber()).to.equal(100); expect(logger.calledWith(LogLevel.warn)).to.be.false; }); } + for (const rowLimit of [2 ** 64, 2 ** 31, 2147483647, -1, Infinity, -Infinity, '18446744073709551616']) { + it(`should reject out-of-range rowLimit ${rowLimit} before sending a request`, async () => { + const context = new ClientContextStub(); + const execute = sinon.spy(context.driver, 'executeStatement'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + + try { + await session.executeStatement('SELECT 1', { rowLimit: rowLimit as number }); + expect.fail('Expected ParameterError'); + } catch (error) { + expect(error).to.be.instanceOf(ParameterError); + expect((error as Error).message).to.equal('rowLimit must be between 0 and 2147483646'); + } + expect(execute.called).to.be.false; + }); + } + + describe('total rowLimit with column-based results', () => { + for (const disableBuffering of [false, true]) { + for (const prefetched of [false, true]) { + for (const rowLimit of [0, 3, 5, 6, 8, undefined]) { + it(`caps ${rowLimit} rows across pages (disableBuffering=${disableBuffering}, prefetched=${prefetched})`, async () => { + const context = new ClientContextStub({ arrowEnabled: false }); + context.driver.executeStatementResp.operationHandle!.hasResultSet = true; + const page = (values: string[]) => ({ + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + columns: [{ stringVal: { values, nulls: Buffer.alloc(0) } }], + }, + }); + const firstPage = { ...page(['a', 'b']), hasMoreRows: true }; + const fetch = sinon.stub(context.driver, 'fetchResults'); + if (prefetched) { + context.driver.executeStatementResp.directResults = { resultSet: firstPage }; + fetch.onFirstCall().resolves({ ...page(['c', 'd']), hasMoreRows: true }); + fetch.onSecondCall().resolves(page(['e', 'f'])); + } else { + fetch.onFirstCall().resolves(firstPage); + fetch.onSecondCall().resolves({ ...page(['c', 'd']), hasMoreRows: true }); + fetch.onThirdCall().resolves(page(['e', 'f'])); + } + fetch.resolves(page([])); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + const operation = await session.executeStatement('SELECT test FROM table', { rowLimit }); + const rows: object[] = []; + while (await operation.hasMoreRows()) { + rows.push(...(await operation.fetchChunk({ maxRows: 2, disableBuffering }))); + } + const expected = ['a', 'b', 'c', 'd', 'e', 'f'].slice(0, rowLimit).map((test) => ({ test })); + expect(rows).to.deep.equal(expected); + expect(await operation.hasMoreRows()).to.be.false; + const fetchCount = fetch.callCount; + expect(await operation.fetchChunk({ maxRows: 2, disableBuffering })).to.deep.equal([]); + expect(await operation.fetchAll()).to.deep.equal([]); + expect(fetch.callCount).to.equal(fetchCount); + if (rowLimit === 0) { + expect(fetchCount).to.equal(0); + } + }); + } + } + } + }); + it('should warn only about statementConf when rowLimit is also set', async () => { const context = new ClientContextStub(); const logger = sinon.spy(context.logger, 'log'); From 2f1696a0934d8f245d4cb967fc949067115b4b68 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 21:06:27 +0000 Subject: [PATCH 4/7] fix(thrift): stop row-limit prefetch and preserve staging instructions Carry the statement budget into Arrow conversion and CloudFetch link scheduling, exempt automatic staging instructions, and preserve numeric fetch-size coercion. Clarify backend-specific validation and cover oversized Arrow pages, failing later pages, bounded downloads and zero-limit staging. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + CONNECTION_PARAMETERS.md | 4 +- lib/contracts/IDBSQLSession.ts | 12 +- lib/result/ArrowResultConverter.ts | 28 +- lib/result/CloudFetchResultHandler.ts | 22 +- lib/thrift-backend/ThriftOperationBackend.ts | 17 +- lib/thrift-backend/ThriftSessionBackend.ts | 10 +- .../thrift-backend/ThriftRowLimit.test.ts | 268 ++++++++++++++++++ 8 files changed, 340 insertions(+), 22 deletions(-) create mode 100644 tests/unit/thrift-backend/ThriftRowLimit.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a141d4d6..78e29a4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Thrift now honors the previously ignored per-statement `rowLimit`, sends the server cap, and enforces the total locally across fetches and all result formats (including legacy columnar overflow). `0` returns no rows, unlike JDBC's unlimited `setMaxRows(0)`; `maxRows` remains fetch-page sizing. +- Capped Thrift Arrow/CloudFetch results stop prefetching once the cap is satisfied, without fetching or downloading beyond it. Internal staging instructions remain readable with `rowLimit: 0` when `stagingAllowedLocalPath` enables a transfer. - Thrift converts `rowLimit` with `Number` (`null`/`NaN` become zero) and truncates toward zero, then rejects values outside `0..2147483646`, including infinities, with `ParameterError` before sending a request. Kernel behavior is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 485bf0fd..04c8590d 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -225,7 +225,9 @@ Thrift also enforces the total cap client-side across fetches and all result for including on servers that ignore this field. It applies `Number` coercion (`null`/`NaN` become zero), truncates toward zero, then requires `0..2147483646`; out-of-range values and infinities throw `ParameterError` before any statement request. Zero -returns no rows. Kernel behavior is unchanged. +returns no rows. Arrow/CloudFetch prefetch stops at the cap. The Thrift client +does not cap internal staging instructions when `stagingAllowedLocalPath` enables +a transfer. Kernel validation happens on the server; its behavior is unchanged. ## Metadata-operation request parameters diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index d6f166aa..a6d09c5c 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -51,11 +51,13 @@ export type ExecuteStatementOptions = { * Per-statement server-side row cap on both backends. Omitted by default (no * driver-imposed cap). Distinct from `maxRows`, which controls fetch-page size. * Thrift also enforces the total cap locally across fetches and result formats. - * It applies Number coercion (numeric strings become numbers, null/NaN become - * zero), then truncates toward zero. The resulting integer must be in - * 0..2147483646; otherwise executeStatement throws ParameterError before sending - * the request. Zero returns no rows (unlike JDBC's unlimited setMaxRows(0)). - * Thrift previously ignored this option. Kernel behavior is unchanged. + * On Thrift, Number coercion applies (numeric strings become numbers, null/NaN + * become zero), followed by truncation toward zero. On Thrift, the resulting + * integer must be in 0..2147483646; otherwise executeStatement throws + * ParameterError before sending the request. The kernel validates on the server. + * Zero returns no rows (unlike JDBC's unlimited setMaxRows(0)). The Thrift client + * exempts internal staging instructions when stagingAllowedLocalPath enables a transfer. + * Thrift previously ignored this option. */ rowLimit?: number; /** diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index f00712f3..b9cbe4ab 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -216,6 +216,8 @@ export default class ArrowResultConverter implements IResultsProvider // Remaining rows in current Arrow batch (not the record batch!) private remainingRows: number = 0; + private remainingRowLimit?: number; + // This is the next (!!) record batch to be read. It is unset only in two cases: // - prior to the first call to `fetchNext` // - when no more data available @@ -236,7 +238,13 @@ export default class ArrowResultConverter implements IResultsProvider preserveBigNumericPrecision = false, disableRowMaterialization = false, rowMode = 'object', - }: { preserveBigNumericPrecision?: boolean; disableRowMaterialization?: boolean; rowMode?: RowMode } = {}, + rowLimit, + }: { + preserveBigNumericPrecision?: boolean; + disableRowMaterialization?: boolean; + rowMode?: RowMode; + rowLimit?: number; + } = {}, ) { this.context = context; this.source = source; @@ -244,13 +252,14 @@ export default class ArrowResultConverter implements IResultsProvider this.preserveBigNumericPrecision = preserveBigNumericPrecision; this.disableRowMaterialization = disableRowMaterialization; this.rowMode = rowMode; + this.remainingRowLimit = rowLimit; if (rowMode === 'object' && !disableRowMaterialization) { warnOnDuplicateColumnNames(context, this.schema); } } public async hasMore() { - if (this.schema.length === 0) { + if (this.schema.length === 0 || this.remainingRowLimit === 0) { return false; } if (this.prefetchedRecordBatch) { @@ -260,7 +269,7 @@ export default class ArrowResultConverter implements IResultsProvider } public async fetchNext(options: ResultsProviderFetchNextOptions) { - if (this.schema.length === 0) { + if (this.schema.length === 0 || this.remainingRowLimit === 0) { return []; } @@ -280,7 +289,7 @@ export default class ArrowResultConverter implements IResultsProvider this.prefetchedRecordBatch = undefined; // Get table rows, but not more than remaining count. When materialization // is disabled, skip `toArray()`/`getRows()` and return dense `null` placeholders. - const batchRows = Math.min(batch.numRows, this.remainingRows); + const batchRows = Math.min(batch.numRows, this.remainingRows, this.remainingRowLimit ?? Infinity); let result: Array; if (this.disableRowMaterialization) { result = new Array(batchRows).fill(null); @@ -288,14 +297,17 @@ export default class ArrowResultConverter implements IResultsProvider result = this.getRowArrays(batch, batchRows); } else { const table = new Table(batch); - result = this.getRows(table.schema, table.toArray().slice(0, this.remainingRows)); + result = this.getRows(table.schema, table.toArray().slice(0, batchRows)); } // Reduce remaining rows count by a count of rows we just processed. // If the remaining count reached zero - we're done with current arrow // batch, so discard the batch reader this.remainingRows -= result.length; - if (this.remainingRows === 0) { + if (this.remainingRowLimit !== undefined) { + this.remainingRowLimit -= result.length; + } + if (this.remainingRows === 0 || this.remainingRowLimit === 0) { this.recordBatchReader = undefined; } @@ -313,6 +325,10 @@ export default class ArrowResultConverter implements IResultsProvider // This method pulls the next item from source if needed, initializes a record batch reader and // gets the next item from it - until either reaches end of data or finds a non-empty record batch private async prefetch(options: ResultsProviderFetchNextOptions) { + if (this.remainingRowLimit === 0) { + return; + } + // This loop will be executed until a next non-empty record batch is retrieved // Another implicit loop condition (end of data) is checked in the loop body while (!this.prefetchedRecordBatch) { diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index fbfe5c7c..2d0a8a0c 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -35,16 +35,20 @@ export default class CloudFetchResultHandler implements IResultsProvider, metadata: TGetResultSetMetadataResp, statementId?: string, + rowLimit?: number, ) { this.context = context; this.source = source; this.isLZ4Compressed = metadata.lz4Compressed ?? false; this.statementId = statementId; + this.remainingRows = rowLimit; if (this.isLZ4Compressed && !LZ4()) { throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4-napi` not installed'); @@ -55,15 +59,23 @@ export default class CloudFetchResultHandler implements IResultsProvider 0 || this.downloadTasks.length > 0) { return true; } - return this.source.hasMore(); + return this.remainingRows !== 0 && this.source.hasMore(); } public async fetchNext(options: ResultsProviderFetchNextOptions) { - const data = await this.source.fetchNext(options); + if (this.remainingRows !== 0) { + const data = await this.source.fetchNext(options); - data?.resultLinks?.forEach((link) => { - this.pendingLinks.push(link); - }); + for (const link of data?.resultLinks ?? []) { + if (this.remainingRows === 0) { + break; + } + this.pendingLinks.push(link); + if (this.remainingRows !== undefined) { + this.remainingRows = Math.max(0, this.remainingRows - link.rowCount.toNumber(true)); + } + } + } const clientConfig = this.context.getConfig(); const freeTaskSlotsCount = clientConfig.cloudFetchConcurrentDownloads - this.downloadTasks.length; diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 05a2b3ec..61646dcd 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -38,6 +38,7 @@ interface ThriftOperationBackendOptions { emptyResultOnNotFound?: TTableSchema; rowMode?: RowMode; rowLimit?: number; + stagingAllowed?: boolean; } async function delay(ms?: number): Promise { @@ -108,6 +109,8 @@ export default class ThriftOperationBackend implements IOperationBackend { private remainingRows?: number; + private readonly stagingAllowed?: boolean; + constructor({ handle, directResults, @@ -116,6 +119,7 @@ export default class ThriftOperationBackend implements IOperationBackend { emptyResultOnNotFound, rowMode, rowLimit, + stagingAllowed, }: ThriftOperationBackendOptions) { this.operationHandle = handle; this.context = context; @@ -123,6 +127,7 @@ export default class ThriftOperationBackend implements IOperationBackend { this.emptyResultOnNotFound = emptyResultOnNotFound; this.rowMode = rowMode; this.remainingRows = rowLimit; + this.stagingAllowed = stagingAllowed; this.metadata = directResults?.resultSetMetadata; const useOnlyPrefetchedResults = Boolean(directResults?.closeOperation); @@ -204,7 +209,7 @@ export default class ThriftOperationBackend implements IOperationBackend { } let rows = await resultHandler.fetchNext({ - limit: this.remainingRows === undefined ? limit : Math.min(limit, this.remainingRows), + limit: this.remainingRows === undefined ? limit : Math.min(Number(limit), this.remainingRows), disableBuffering, }); if (this.remainingRows !== undefined) { @@ -314,7 +319,11 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async getResultMetadata(): Promise { - return this.adaptResultMetadata(await this.thriftResultMetadataResponse()); + const metadata = await this.thriftResultMetadataResponse(); + if (this.stagingAllowed && metadata.isStagingOperation) { + this.remainingRows = undefined; + } + return this.adaptResultMetadata(metadata); } public getCachedResultMetadata(): ResultMetadata | undefined { @@ -408,18 +417,20 @@ export default class ThriftOperationBackend implements IOperationBackend { preserveBigNumericPrecision: this.context.getConfig().preserveBigNumericPrecision ?? false, disableRowMaterialization: this.context.getConfig().disableRowMaterialization ?? false, rowMode: this.rowMode, + rowLimit: this.remainingRows, }, ); break; case TSparkRowSetType.URL_BASED_SET: resultSource = new ArrowResultConverter( this.context, - new CloudFetchResultHandler(this.context, this._data, metadata, this.id), + new CloudFetchResultHandler(this.context, this._data, metadata, this.id, this.remainingRows), metadata, { preserveBigNumericPrecision: this.context.getConfig().preserveBigNumericPrecision ?? false, disableRowMaterialization: this.context.getConfig().disableRowMaterialization ?? false, rowMode: this.rowMode, + rowLimit: this.remainingRows, }, ); break; diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index fc6523a7..8a989bcb 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -227,7 +227,11 @@ export default class ThriftSessionBackend implements ISessionBackend { } const response = await driver.executeStatement(request); - return this.createOperationBackend(response, { rowMode: options.rowMode, rowLimit }); + return this.createOperationBackend(response, { + rowMode: options.rowMode, + rowLimit, + stagingAllowed: options.stagingAllowedLocalPath !== undefined, + }); } public async getTypeInfo(request: TypeInfoRequest): Promise { @@ -382,7 +386,8 @@ export default class ThriftSessionBackend implements ISessionBackend { functionCatalog, rowMode, rowLimit, - }: { functionCatalog?: string | null; rowMode?: RowMode; rowLimit?: number } = {}, + stagingAllowed, + }: { functionCatalog?: string | null; rowMode?: RowMode; rowLimit?: number; stagingAllowed?: boolean } = {}, ): IOperationBackend { Status.assert(response.status); const handle = definedOrError(response.operationHandle); @@ -393,6 +398,7 @@ export default class ThriftSessionBackend implements ISessionBackend { functionCatalog, rowMode, rowLimit, + stagingAllowed, }); } } diff --git a/tests/unit/thrift-backend/ThriftRowLimit.test.ts b/tests/unit/thrift-backend/ThriftRowLimit.test.ts new file mode 100644 index 00000000..f2628d24 --- /dev/null +++ b/tests/unit/thrift-backend/ThriftRowLimit.test.ts @@ -0,0 +1,268 @@ +import assert from 'assert'; +import http from 'http'; +import { AddressInfo } from 'net'; +import { expect } from 'chai'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import Int64 from 'node-int64'; +import sinon from 'sinon'; +import { + TFetchResultsResp, + TSparkRowSetType, + TSessionHandle, + TStatusCode, + TTypeId, +} from '../../../thrift/TCLIService_types'; +import { RowMode } from '../../../lib/contracts/IDBSQLSession'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import { createSessionForTest } from '../.stubs/createSessionForTest'; + +const sessionHandle: TSessionHandle = { + sessionId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, +}; +const status = { statusCode: TStatusCode.SUCCESS_STATUS }; + +function createArrowPage( + start: number, + hasMoreRows: boolean, +): { schema: Buffer; ipc: Buffer; page: TFetchResultsResp } { + const table = tableFromArrays({ id: new Int32Array(Array.from({ length: 6 }, (_, index) => start + index)) }); + const ipc = Buffer.from(tableToIPC(table, 'stream')); + const schemaLength = 8 + ipc.readInt32LE(4); + return { + schema: ipc.subarray(0, schemaLength), + ipc, + page: { + status, + hasMoreRows, + results: { + startRowOffset: new Int64(start), + rows: [], + arrowBatches: [{ batch: ipc.subarray(schemaLength), rowCount: new Int64(6) }], + }, + }, + }; +} + +function createContext(resultFormat: TSparkRowSetType): ClientContextStub { + const context = new ClientContextStub(); + context.driver.executeStatementResp.operationHandle!.hasResultSet = true; + context.driver.getResultSetMetadataResp.resultFormat = resultFormat; + context.driver.getResultSetMetadataResp.schema = { + columns: [{ columnName: 'id', position: 1, typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] } }], + }; + return context; +} + +describe('Thrift rowLimit source budgets', () => { + afterEach(() => sinon.restore()); + + for (const disableBuffering of [false, true]) { + for (const prefetched of [false, true]) { + for (const rowLimit of [0, 4, 6, 9, 12, 20, undefined]) { + it(`caps oversized Arrow pages at ${rowLimit} (disableBuffering=${disableBuffering}, prefetched=${prefetched})`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + const first = createArrowPage(0, true); + const second = createArrowPage(6, false); + context.driver.getResultSetMetadataResp.arrowSchema = first.schema; + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(new Error('unexpected fetch past cap')); + if (prefetched) { + context.driver.executeStatementResp.directResults = { + resultSetMetadata: context.driver.getResultSetMetadataResp, + resultSet: first.page, + }; + fetch.onFirstCall().resolves(second.page); + } else { + fetch.onFirstCall().resolves(first.page); + fetch.onSecondCall().resolves(second.page); + } + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(12)', { rowLimit }); + const rows: object[] = []; + while (await operation.hasMoreRows()) { + rows.push(...(await operation.fetchChunk({ maxRows: 2, disableBuffering }))); + } + const count = Math.min(rowLimit ?? 12, 12); + expect(rows).to.deep.equal(Array.from({ length: count }, (_, id) => ({ id }))); + expect(await operation.hasMoreRows()).to.be.false; + expect(await operation.fetchAll()).to.deep.equal([]); + expect(await operation.fetchChunk()).to.deep.equal([]); + const pages = Math.ceil(count / 6); + expect(fetch.callCount).to.equal(Math.max(0, pages - (prefetched ? 1 : 0))); + }); + } + } + } + + for (const rowLimit of [4, 6, 9]) { + it(`does not read a failing Arrow page beyond cap ${rowLimit}`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + const first = createArrowPage(0, true); + context.driver.getResultSetMetadataResp.arrowSchema = first.schema; + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(new Error('required page failed')); + fetch.onFirstCall().resolves(first.page); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(12)', { rowLimit }); + if (rowLimit > 6) { + await assert.rejects(operation.fetchAll(), /required page failed/); + } else { + expect(await operation.fetchAll()).to.deep.equal(Array.from({ length: rowLimit }, (_, id) => ({ id }))); + expect(await operation.hasMoreRows()).to.be.false; + expect(fetch.callCount).to.equal(1); + } + }); + } + + it('coerces a BigInt fetch-page limit when a statement cap is set', async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + const first = createArrowPage(0, false); + context.driver.getResultSetMetadataResp.arrowSchema = first.schema; + sinon.stub(context.driver, 'fetchResults').resolves(first.page); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(6)', { rowLimit: 5 }); + expect(await operation.fetchChunk({ maxRows: BigInt(2) as unknown as number })).to.deep.equal([ + { id: 0 }, + { id: 1 }, + ]); + expect(await operation.fetchAll()).to.deep.equal([{ id: 2 }, { id: 3 }, { id: 4 }]); + }); + + for (const rowMode of ['object', 'array'] as RowMode[]) { + for (const disableRowMaterialization of [false, true]) { + it(`caps Arrow rowMode=${rowMode} with disableRowMaterialization=${disableRowMaterialization}`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + context.configOverrides.disableRowMaterialization = disableRowMaterialization; + const first = createArrowPage(0, true); + context.driver.getResultSetMetadataResp.arrowSchema = first.schema; + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(new Error('unexpected prefetch')); + fetch.onFirstCall().resolves(first.page); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(6)', { rowLimit: 4, rowMode }); + expect(await operation.fetchAll()).to.deep.equal( + Array.from({ length: 4 }, (_, id) => { + if (disableRowMaterialization) { + return null; + } + return rowMode === 'array' ? [id] : { id }; + }), + ); + expect(fetch.callCount).to.equal(1); + }); + } + } + + describe('CloudFetch and staging', () => { + let server: http.Server; + let baseUrl: string; + let requests: string[]; + + before(async () => { + server = http.createServer((request, response) => { + requests.push(`${request.method} ${request.url}`); + if (request.method === 'DELETE') { + response.end(); + } else { + const start = Number(request.url!.slice(1)) * 6; + response.end(createArrowPage(start, false).ipc); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + beforeEach(() => { + requests = []; + }); + + after(async () => { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + }); + + for (const concurrency of [1, 16]) { + for (const disableBuffering of [false, true]) { + for (const rowLimit of [0, 4, 6, 9, 12, 20, undefined]) { + it(`bounds CloudFetch downloads at ${rowLimit} (concurrency=${concurrency}, disableBuffering=${disableBuffering})`, async () => { + const context = createContext(TSparkRowSetType.URL_BASED_SET); + context.configOverrides.cloudFetchConcurrentDownloads = concurrency; + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(new Error('unexpected fetch past cap')); + fetch.onFirstCall().resolves({ + status, + hasMoreRows: rowLimit !== undefined && rowLimit <= 12, + results: { + startRowOffset: new Int64(0), + rows: [], + resultLinks: Array.from({ length: 3 }, (_, index) => ({ + fileLink: `${baseUrl}/${index}`, + expiryTime: new Int64(Date.now() + 60000), + startRowOffset: new Int64(index * 6), + rowCount: new Int64(6), + bytesNum: new Int64(0), + })), + }, + }); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(18)', { rowLimit }); + const rows: object[] = []; + while (await operation.hasMoreRows()) { + rows.push(...(await operation.fetchChunk({ maxRows: 2, disableBuffering }))); + } + const count = Math.min(rowLimit ?? 18, 18); + expect(rows).to.deep.equal(Array.from({ length: count }, (_, id) => ({ id }))); + expect(await operation.fetchChunk()).to.deep.equal([]); + expect(await operation.hasMoreRows()).to.be.false; + expect(fetch.callCount).to.equal(count === 0 ? 0 : 1); + expect(requests).to.have.members( + Array.from({ length: Math.ceil(count / 6) }, (_, index) => `GET /${index}`), + ); + }); + } + } + } + + for (const rowMode of ['object', 'array'] as RowMode[]) { + for (const stagingAllowed of [false, true]) { + it(`preserves internal staging instructions with rowLimit=0 (rowMode=${rowMode}, stagingAllowed=${stagingAllowed})`, async () => { + const context = createContext(TSparkRowSetType.COLUMN_BASED_SET); + context.driver.getResultSetMetadataResp.isStagingOperation = true; + context.driver.getResultSetMetadataResp.schema = { + columns: ['presignedUrl', 'operation'].map((columnName, index) => ({ + columnName, + position: index + 1, + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.STRING_TYPE } }] }, + })), + }; + const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ status, hasMoreRows: false }); + fetch.onFirstCall().resolves({ + status, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + columns: [ + { stringVal: { values: [`${baseUrl}/staging`], nulls: Buffer.alloc(0) } }, + { stringVal: { values: ['REMOVE'], nulls: Buffer.alloc(0) } }, + ], + }, + }); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('REMOVE volume_file', { + rowLimit: 0, + stagingAllowedLocalPath: stagingAllowed ? '/tmp' : undefined, + rowMode, + }); + expect(await operation.fetchAll()).to.deep.equal([]); + expect(requests).to.deep.equal(stagingAllowed ? ['DELETE /staging'] : []); + }); + } + } + + it('keeps ordinary results capped when stagingAllowedLocalPath is set', async () => { + const context = createContext(TSparkRowSetType.COLUMN_BASED_SET); + const fetch = sinon.spy(context.driver, 'fetchResults'); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT 1', { rowLimit: 0, stagingAllowedLocalPath: '/tmp' }); + expect(await operation.fetchAll()).to.deep.equal([]); + expect(fetch.called).to.be.false; + expect(requests).to.deep.equal([]); + }); + }); +}); From 17153bbd2f2896f9a973bc95c4bab4effc74ae5a Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 21:44:44 +0000 Subject: [PATCH 5/7] fix(thrift): reserve server rows for automatic staging instructions Floor an explicitly set wire row limit at one when staging is enabled, while retaining the requested client cap for ordinary results. Cover an Arrow server that honors the wire cap and a default-Arrow live transfer. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 9 +- lib/contracts/IDBSQLSession.ts | 7 +- lib/thrift-backend/ThriftSessionBackend.ts | 5 +- tests/e2e/staging_ingestion.test.ts | 48 ++++++++++ tests/unit/DBSQLSession.test.ts | 18 ++++ .../thrift-backend/ThriftRowLimit.test.ts | 90 +++++++++++++++++++ 7 files changed, 172 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e29a4b..b1751bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased - Thrift now honors the previously ignored per-statement `rowLimit`, sends the server cap, and enforces the total locally across fetches and all result formats (including legacy columnar overflow). `0` returns no rows, unlike JDBC's unlimited `setMaxRows(0)`; `maxRows` remains fetch-page sizing. -- Capped Thrift Arrow/CloudFetch results stop prefetching once the cap is satisfied, without fetching or downloading beyond it. Internal staging instructions remain readable with `rowLimit: 0` when `stagingAllowedLocalPath` enables a transfer. +- Capped Thrift Arrow/CloudFetch results stop prefetching once the cap is satisfied, without fetching or downloading beyond it. With `stagingAllowedLocalPath`, Thrift sends at least one for an explicitly set, normalized `rowLimit` so the server preserves the staging instruction row, including on Arrow. The client still caps ordinary results at the requested limit and exempts internal staging instructions; an unset limit remains omitted. - Thrift converts `rowLimit` with `Number` (`null`/`NaN` become zero) and truncates toward zero, then rejects values outside `0..2147483646`, including infinities, with `ParameterError` before sending a request. Kernel behavior is unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index 04c8590d..0bce4469 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -225,9 +225,12 @@ Thrift also enforces the total cap client-side across fetches and all result for including on servers that ignore this field. It applies `Number` coercion (`null`/`NaN` become zero), truncates toward zero, then requires `0..2147483646`; out-of-range values and infinities throw `ParameterError` before any statement request. Zero -returns no rows. Arrow/CloudFetch prefetch stops at the cap. The Thrift client -does not cap internal staging instructions when `stagingAllowedLocalPath` enables -a transfer. Kernel validation happens on the server; its behavior is unchanged. +returns no rows. Arrow/CloudFetch prefetch stops at the cap. When +`stagingAllowedLocalPath` is set, Thrift sends `Math.max(normalizedRowLimit, 1)` +for an explicitly set limit so the server preserves the staging instruction row; +an unset limit remains omitted. The Thrift client still caps ordinary results at the +requested limit and exempts internal staging instructions from that cap. Kernel +validation happens on the server; its behavior is unchanged. ## Metadata-operation request parameters diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index a6d09c5c..2650692e 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -55,8 +55,11 @@ export type ExecuteStatementOptions = { * become zero), followed by truncation toward zero. On Thrift, the resulting * integer must be in 0..2147483646; otherwise executeStatement throws * ParameterError before sending the request. The kernel validates on the server. - * Zero returns no rows (unlike JDBC's unlimited setMaxRows(0)). The Thrift client - * exempts internal staging instructions when stagingAllowedLocalPath enables a transfer. + * Zero returns no rows (unlike JDBC's unlimited setMaxRows(0)). When + * stagingAllowedLocalPath is set, Thrift sends at least one for a set rowLimit + * to preserve the staging instruction row; an unset limit remains omitted. + * The Thrift client still caps ordinary results at the requested limit and exempts + * internal staging instructions from that cap. * Thrift previously ignored this option. */ rowLimit?: number; diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 8a989bcb..2a873fb0 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -203,7 +203,10 @@ export default class ThriftSessionBackend implements ISessionBackend { sessionHandle: this.sessionHandle, statement, queryTimeout: options.queryTimeout ? numberToInt64(options.queryTimeout) : undefined, - resultRowLimit: rowLimit !== undefined ? numberToInt64(rowLimit) : undefined, + resultRowLimit: + rowLimit !== undefined + ? numberToInt64(options.stagingAllowedLocalPath !== undefined ? Math.max(rowLimit, 1) : rowLimit) + : undefined, runAsync: true, ...getDirectResultsOptions(options.maxRows, clientConfig), ...getArrowOptions(clientConfig, this.serverProtocolVersion), diff --git a/tests/e2e/staging_ingestion.test.ts b/tests/e2e/staging_ingestion.test.ts index 7721dc75..02e07cb4 100644 --- a/tests/e2e/staging_ingestion.test.ts +++ b/tests/e2e/staging_ingestion.test.ts @@ -1,10 +1,12 @@ import { expect } from 'chai'; +import assert from 'assert'; import fs from 'fs'; import path from 'path'; import os from 'os'; import * as uuid from 'uuid'; import { DBSQLClient } from '../../lib'; import StagingError from '../../lib/errors/StagingError'; +import { ResultFormat } from '../../lib/contracts/ResultMetadata'; import config from './utils/config'; @@ -95,6 +97,52 @@ describe('Staging Test', () => { } }); + it('put, get and remove staging data with zero rowLimit and default Arrow', async () => { + const { catalog, schema, volume } = config; + const client = new DBSQLClient(); + expect(client.getConfig().arrowEnabled).to.be.true; + const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; + const localFile = path.join(localPath, `${uuid.v4()}.csv`); + const expectedData = 'zero-row cap staging regression'; + const stagingOptions = { stagingAllowedLocalPath: [localPath] }; + const options = { ...stagingOptions, rowLimit: 0 }; + fs.writeFileSync(localFile, expectedData); + try { + await client.connect({ host: config.host, path: config.path, token: config.token }); + const session = await client.openSession({ initialCatalog: catalog, initialSchema: schema }); + try { + const put = await session.executeStatement(`PUT '${localFile}' INTO '${stagingFileName}' OVERWRITE`, options); + try { + expect((await put.getResultMetadata()).resultFormat).to.equal(ResultFormat.ArrowBased); + } finally { + await put.close(); + } + fs.rmSync(localFile); + const get = await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, options); + try { + expect(fs.readFileSync(localFile, 'utf8')).to.equal(expectedData); + } finally { + await get.close(); + } + const remove = await session.executeStatement(`REMOVE '${stagingFileName}'`, options); + await remove.close(); + await assert.rejects( + session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, stagingOptions), + (error: unknown) => error instanceof StagingError && error.message.includes('404'), + ); + } finally { + try { + const cleanup = await session.executeStatement(`REMOVE '${stagingFileName}'`, stagingOptions); + await cleanup.close(); + } finally { + await session.close(); + } + } + } finally { + await client.close(); + } + }); + it('delete non-existent data', async () => { const { catalog, schema, volume } = config; diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 7581b292..0c61480d 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -106,6 +106,24 @@ describe('DBSQLSession', () => { }); } + for (const rowLimit of [0, 1, 7, 2147483646, undefined, null, NaN, 3.9]) { + it(`reserves a staging instruction row on the wire for rowLimit ${rowLimit}`, async () => { + const context = new ClientContextStub(); + const execute = sinon.spy(context.driver, 'executeStatement'); + const session = createSessionForTest({ handle: sessionHandleStub, context }); + await session.executeStatement('SELECT 1', { + rowLimit: rowLimit as number | undefined, + stagingAllowedLocalPath: '/tmp', + }); + const wireLimit = execute.firstCall.args[0].resultRowLimit; + if (rowLimit === undefined) { + expect(wireLimit).to.not.exist; + } else { + expect(wireLimit?.toNumber()).to.equal(Math.max(Math.trunc(Number(rowLimit) || 0), 1)); + } + }); + } + for (const rowLimit of [2 ** 64, 2 ** 31, 2147483647, -1, Infinity, -Infinity, '18446744073709551616']) { it(`should reject out-of-range rowLimit ${rowLimit} before sending a request`, async () => { const context = new ClientContextStub(); diff --git a/tests/unit/thrift-backend/ThriftRowLimit.test.ts b/tests/unit/thrift-backend/ThriftRowLimit.test.ts index f2628d24..aa93c724 100644 --- a/tests/unit/thrift-backend/ThriftRowLimit.test.ts +++ b/tests/unit/thrift-backend/ThriftRowLimit.test.ts @@ -255,6 +255,96 @@ describe('Thrift rowLimit source budgets', () => { } } + for (const prefetched of [false, true]) { + for (const rowMode of ['object', 'array'] as RowMode[]) { + it(`reads an Arrow staging instruction when the server honors the cap (prefetched=${prefetched}, rowMode=${rowMode})`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + const metadata = context.driver.getResultSetMetadataResp; + metadata.isStagingOperation = true; + metadata.schema = { + columns: ['presignedUrl', 'operation'].map((columnName, index) => ({ + columnName, + position: index + 1, + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.STRING_TYPE } }] }, + })), + }; + const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ status, hasMoreRows: false }); + const execute = sinon.stub(context.driver, 'executeStatement').callsFake(async (request) => { + const table = tableFromArrays({ presignedUrl: [`${baseUrl}/staging`], operation: ['REMOVE'] }).slice( + 0, + request.resultRowLimit?.toNumber() ?? 1, + ); + const ipc = Buffer.from(tableToIPC(table, 'stream')); + const schemaLength = 8 + ipc.readInt32LE(4); + metadata.arrowSchema = ipc.subarray(0, schemaLength); + const page: TFetchResultsResp = { + status, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: ipc.subarray(schemaLength), rowCount: new Int64(table.numRows) }], + }, + }; + if (prefetched) { + context.driver.executeStatementResp.directResults = { resultSet: page, resultSetMetadata: metadata }; + } else { + fetch.onFirstCall().resolves(page); + } + return context.driver.executeStatementResp; + }); + const session = createSessionForTest({ handle: sessionHandle, context }); + try { + const operation = await session.executeStatement('REMOVE volume_file', { + rowLimit: 0, + stagingAllowedLocalPath: '/tmp', + rowMode, + }); + expect(execute.firstCall.args[0].resultRowLimit?.toNumber()).to.equal(1); + expect(requests).to.deep.equal(['DELETE /staging']); + expect(await operation.fetchAll()).to.deep.equal([]); + } finally { + await session.close(); + } + }); + } + } + + for (const rowLimit of [0, 3]) { + it(`keeps an ordinary Arrow result capped at ${rowLimit} with stagingAllowedLocalPath set`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ status, hasMoreRows: false }); + const execute = sinon.stub(context.driver, 'executeStatement').callsFake(async (request) => { + const table = tableFromArrays({ id: new Int32Array([0, 1, 2, 3, 4, 5]) }).slice( + 0, + request.resultRowLimit?.toNumber() ?? 6, + ); + const ipc = Buffer.from(tableToIPC(table, 'stream')); + const schemaLength = 8 + ipc.readInt32LE(4); + context.driver.getResultSetMetadataResp.arrowSchema = ipc.subarray(0, schemaLength); + fetch.onFirstCall().resolves({ + status, + hasMoreRows: false, + results: { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: ipc.subarray(schemaLength), rowCount: new Int64(table.numRows) }], + }, + }); + return context.driver.executeStatementResp; + }); + const session = createSessionForTest({ handle: sessionHandle, context }); + const operation = await session.executeStatement('SELECT id FROM range(6)', { + rowLimit, + stagingAllowedLocalPath: '/tmp', + }); + expect(execute.firstCall.args[0].resultRowLimit?.toNumber()).to.equal(Math.max(rowLimit, 1)); + expect(await operation.fetchAll()).to.deep.equal(Array.from({ length: rowLimit }, (_, id) => ({ id }))); + expect(await operation.hasMoreRows()).to.be.false; + expect(requests).to.deep.equal([]); + }); + } + it('keeps ordinary results capped when stagingAllowedLocalPath is set', async () => { const context = createContext(TSparkRowSetType.COLUMN_BASED_SET); const fetch = sinon.spy(context.driver, 'fetchResults'); From 390bf965fa2bcc1ef621cc6085a21b1b2f88fb42 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 21:54:26 +0000 Subject: [PATCH 6/7] fix(thrift): preserve prefetched failures with a zero row cap Validate an already-received direct-results status before the zero-cap shortcuts can skip it. Cover both fetchAll and hasMoreRows, without another fetch, and add the PR reference to the row-limit changelog entries. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 6 ++--- lib/thrift-backend/ThriftOperationBackend.ts | 4 ++++ .../thrift-backend/ThriftRowLimit.test.ts | 24 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1751bc7..9e52e9f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Unreleased -- Thrift now honors the previously ignored per-statement `rowLimit`, sends the server cap, and enforces the total locally across fetches and all result formats (including legacy columnar overflow). `0` returns no rows, unlike JDBC's unlimited `setMaxRows(0)`; `maxRows` remains fetch-page sizing. -- Capped Thrift Arrow/CloudFetch results stop prefetching once the cap is satisfied, without fetching or downloading beyond it. With `stagingAllowedLocalPath`, Thrift sends at least one for an explicitly set, normalized `rowLimit` so the server preserves the staging instruction row, including on Arrow. The client still caps ordinary results at the requested limit and exempts internal staging instructions; an unset limit remains omitted. -- Thrift converts `rowLimit` with `Number` (`null`/`NaN` become zero) and truncates toward zero, then rejects values outside `0..2147483646`, including infinities, with `ParameterError` before sending a request. Kernel behavior is unchanged. +- Thrift now honors the previously ignored per-statement `rowLimit`, sends the server cap, and enforces the total locally across fetches and all result formats (including legacy columnar overflow). `0` returns no rows, unlike JDBC's unlimited `setMaxRows(0)`; `maxRows` remains fetch-page sizing. (#623) +- Capped Thrift Arrow/CloudFetch results stop prefetching once the cap is satisfied, without fetching or downloading beyond it. With `stagingAllowedLocalPath`, Thrift sends at least one for an explicitly set, normalized `rowLimit` so the server preserves the staging instruction row, including on Arrow. The client still caps ordinary results at the requested limit and exempts internal staging instructions; an unset limit remains omitted. (#623) +- Thrift converts `rowLimit` with `Number` (`null`/`NaN` become zero) and truncates toward zero, then rejects values outside `0..2147483646`, including infinities, with `ParameterError` before sending a request. Kernel behavior is unchanged. (#623) - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 61646dcd..2aa8bbfe 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -136,6 +136,10 @@ export default class ThriftOperationBackend implements IOperationBackend { this.processOperationStatusResponse(directResults.operationStatus); } + if (rowLimit === 0 && directResults?.resultSet) { + Status.assert(directResults.resultSet.status); + } + this._data = new RowSetProvider( this.context, this.operationHandle, diff --git a/tests/unit/thrift-backend/ThriftRowLimit.test.ts b/tests/unit/thrift-backend/ThriftRowLimit.test.ts index aa93c724..e2f6fbbf 100644 --- a/tests/unit/thrift-backend/ThriftRowLimit.test.ts +++ b/tests/unit/thrift-backend/ThriftRowLimit.test.ts @@ -13,6 +13,7 @@ import { TTypeId, } from '../../../thrift/TCLIService_types'; import { RowMode } from '../../../lib/contracts/IDBSQLSession'; +import StatusError from '../../../lib/errors/StatusError'; import ClientContextStub from '../.stubs/ClientContextStub'; import { createSessionForTest } from '../.stubs/createSessionForTest'; @@ -56,6 +57,29 @@ function createContext(resultFormat: TSparkRowSetType): ClientContextStub { describe('Thrift rowLimit source budgets', () => { afterEach(() => sinon.restore()); + for (const method of ['fetchAll', 'hasMoreRows'] as const) { + it(`reports an already-prefetched error before zero-cap ${method}`, async () => { + const context = createContext(TSparkRowSetType.ARROW_BASED_SET); + context.driver.executeStatementResp.directResults = { + resultSet: { status: { statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'direct fetch failed' } }, + }; + const fetch = sinon.spy(context.driver, 'fetchResults'); + const session = createSessionForTest({ handle: sessionHandle, context }); + try { + await assert.rejects( + async () => { + const operation = await session.executeStatement('SELECT id FROM range(6)', { rowLimit: 0 }); + await operation[method](); + }, + (error: unknown) => error instanceof StatusError && error.message === 'direct fetch failed', + ); + expect(fetch.called).to.be.false; + } finally { + await session.close(); + } + }); + } + for (const disableBuffering of [false, true]) { for (const prefetched of [false, true]) { for (const rowLimit of [0, 4, 6, 9, 12, 20, undefined]) { From 94420a8dd6899861d7fb4fb88b53020ba0fa6905 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 18 Nov 2026 21:59:45 +0000 Subject: [PATCH 7/7] fix(thrift): defer capped direct-result validation until fetch Validate queued response statuses through the existing row-set provider only when an exhausted operation is read. Preserve the operation handle for callers to close after a failed fetch, with lifecycle regressions for fetchAll and hasMoreRows. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/result/RowSetProvider.ts | 4 ++++ lib/thrift-backend/ThriftOperationBackend.ts | 6 ++---- .../thrift-backend/ThriftRowLimit.test.ts | 20 +++++++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 8c67a9f4..cdee2c2a 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -125,6 +125,10 @@ export default class RowSetProvider implements IResultsProvider Status.assert(response.status)); + } + private processFetchResponse(response: TFetchResultsResp): TRowSet | undefined { Status.assert(response.status); this.fetchOrientation = TFetchOrientation.FETCH_NEXT; diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 2aa8bbfe..97ffb41b 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -136,10 +136,6 @@ export default class ThriftOperationBackend implements IOperationBackend { this.processOperationStatusResponse(directResults.operationStatus); } - if (rowLimit === 0 && directResults?.resultSet) { - Status.assert(directResults.resultSet.status); - } - this._data = new RowSetProvider( this.context, this.operationHandle, @@ -183,6 +179,7 @@ export default class ThriftOperationBackend implements IOperationBackend { isClosed?: () => boolean; }): Promise> { if (this.remainingRows === 0) { + this._data.validatePrefetchedResults(); return []; } @@ -232,6 +229,7 @@ export default class ThriftOperationBackend implements IOperationBackend { public async hasMore(): Promise { if (this.remainingRows === 0) { + this._data.validatePrefetchedResults(); return false; } diff --git a/tests/unit/thrift-backend/ThriftRowLimit.test.ts b/tests/unit/thrift-backend/ThriftRowLimit.test.ts index e2f6fbbf..e0e21872 100644 --- a/tests/unit/thrift-backend/ThriftRowLimit.test.ts +++ b/tests/unit/thrift-backend/ThriftRowLimit.test.ts @@ -64,16 +64,20 @@ describe('Thrift rowLimit source budgets', () => { resultSet: { status: { statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'direct fetch failed' } }, }; const fetch = sinon.spy(context.driver, 'fetchResults'); + const close = sinon.spy(context.driver, 'closeOperation'); const session = createSessionForTest({ handle: sessionHandle, context }); try { - await assert.rejects( - async () => { - const operation = await session.executeStatement('SELECT id FROM range(6)', { rowLimit: 0 }); - await operation[method](); - }, - (error: unknown) => error instanceof StatusError && error.message === 'direct fetch failed', - ); - expect(fetch.called).to.be.false; + const operation = await session.executeStatement('SELECT id FROM range(6)', { rowLimit: 0 }); + try { + await assert.rejects( + operation[method](), + (error: unknown) => error instanceof StatusError && error.message === 'direct fetch failed', + ); + expect(fetch.called).to.be.false; + } finally { + await operation.close(); + } + expect(close.calledOnce).to.be.true; } finally { await session.close(); }