From 14c088aa32d8c70fd9aaaf9538fd6a5aa591f985 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 07:34:08 +0000 Subject: [PATCH 1/5] fix(thrift): fail operations after unrecoverable fetch errors Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 + lib/thrift-backend/ThriftOperationBackend.ts | 31 ++++- .../unit/thrift-backend/fetchFailure.test.ts | 131 ++++++++++++++++++ 3 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 tests/unit/thrift-backend/fetchFailure.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..06a664b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- After an unrecoverable Thrift result fetch or CloudFetch download error, the operation is failed and best-effort closed. Later reads throw `OperationStateError` instead of silently returning an incomplete suffix; invalid fetch arguments do not fail the operation. Kernel behavior is unchanged. + - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 10f2d03e..11c740b4 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -105,6 +105,8 @@ export default class ThriftOperationBackend implements IOperationBackend { private resultHandler?: ResultSlicer; + private fetchFailed = false; + constructor({ handle, directResults, @@ -168,6 +170,10 @@ export default class ThriftOperationBackend implements IOperationBackend { disableBuffering?: boolean; isClosed?: () => boolean; }): Promise> { + this.failIfFetchFailed(); + if (Math.abs(limit) > 2 ** 64) { + throw new RangeError('maxRows is outside Int64 range'); + } const resultHandler = await this.getResultHandler(); // All the library code is Promise-based, however, since Promises are microtasks, @@ -194,20 +200,33 @@ export default class ThriftOperationBackend implements IOperationBackend { return []; } - const rows = await resultHandler.fetchNext({ limit, disableBuffering }); - if (this.functionCatalog === undefined) { - return rows; - } + try { + const rows = await resultHandler.fetchNext({ limit, disableBuffering }); + if (this.functionCatalog === undefined) { + return rows; + } - // Native GetFunctions leaves FUNCTION_CAT empty; match JDBC and kernel. - return rows.map((row) => (row === null ? row : { ...row, FUNCTION_CAT: this.functionCatalog })); + // Native GetFunctions leaves FUNCTION_CAT empty; match JDBC and kernel. + return rows.map((row) => (row === null ? row : { ...row, FUNCTION_CAT: this.functionCatalog })); + } catch (error) { + this.fetchFailed = true; + await this.close().catch(() => undefined); + throw error; + } } public async hasMore(): Promise { + this.failIfFetchFailed(); const resultHandler = await this.getResultHandler(); return resultHandler.hasMore(); } + private failIfFetchFailed(): void { + if (this.fetchFailed) { + throw new OperationStateError(OperationStateErrorCode.Error); + } + } + public async status(progress: boolean): Promise { const response = await this.thriftStatusResponse(progress); return this.adaptOperationStatus(response); diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts new file mode 100644 index 00000000..564d7509 --- /dev/null +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -0,0 +1,131 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { TOperationState, TOperationType, TSparkRowSetType } from '../../../thrift/TCLIService_types'; +import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; +import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import { createOperationForTest } from '../.stubs/createOperationForTest'; + +describe('Thrift fetch failure', () => { + function setup() { + const context = new ClientContextStub(); + context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; + const operation = createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + }); + return { context, operation }; + } + + for (const closeFails of [false, true]) { + it(`rejects later reads after an exhausted fetch, even when close ${ + closeFails ? 'fails' : 'succeeds' + }`, async () => { + const { context, operation } = setup(); + const fetchError = new Error('exhausted fetch retries'); + const fetch = sinon.stub(context.driver, 'fetchResults').callThrough(); + fetch.onFirstCall().rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + if (closeFails) { + close.rejects(new Error('close failed')); + } + + await operation.fetchChunk({ maxRows: 3 }).then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + + for (const read of [ + () => operation.fetchChunk({ maxRows: 3 }), + () => operation.fetchAll(), + () => operation.hasMoreRows(), + ]) { + await read().then( + () => expect.fail('later reads must not recover a suffix'), + (error) => { + expect(error).to.be.instanceOf(OperationStateError); + expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + }, + ); + } + expect(fetch.callCount).to.equal(1); + expect(close.callCount).to.equal(1); + }); + } + + it('keeps the operation usable after an invalid maxRows', async () => { + const { context, operation } = setup(); + const close = sinon.spy(context.driver, 'closeOperation'); + const fetch = sinon.spy(context.driver, 'fetchResults'); + + await operation.fetchChunk({ maxRows: Number.MAX_VALUE }).then( + () => expect.fail('maxRows should be outside the Int64 range'), + (error) => expect(error).to.be.instanceOf(RangeError), + ); + expect(close.called).to.be.false; + expect(fetch.called).to.be.false; + expect(await operation.fetchChunk({ maxRows: 3 })).to.deep.equal([{ test: 'a' }, { test: 'b' }, { test: 'c' }]); + }); + + it('does not consume buffered rows when maxRows is invalid', async () => { + const { context, operation } = setup(); + const close = sinon.spy(context.driver, 'closeOperation'); + expect(await operation.fetchChunk({ maxRows: 1 })).to.deep.equal([{ test: 'a' }]); + await operation.fetchChunk({ maxRows: Number.MAX_VALUE }).then( + () => expect.fail('maxRows should be outside the Int64 range'), + (error) => expect(error).to.be.instanceOf(RangeError), + ); + expect(close.called).to.be.false; + expect(await operation.fetchChunk({ maxRows: 2 })).to.deep.equal([{ test: 'b' }, { test: 'c' }]); + }); + + it('fails after a CloudFetch provider error during fetchAll', async () => { + const { context, operation } = setup(); + context.driver.getResultSetMetadataResp.resultFormat = TSparkRowSetType.URL_BASED_SET; + const close = sinon.spy(context.driver, 'closeOperation'); + const downloadError = new RangeError('invalid downloaded Arrow data'); + const download = sinon.stub(CloudFetchResultHandler.prototype, 'fetchNext').rejects(downloadError); + try { + await operation.fetchAll().then( + () => expect.fail('fetchAll should fail'), + (error) => expect(error).to.equal(downloadError), + ); + await operation.fetchChunk().then( + () => expect.fail('later reads must fail'), + (error) => { + expect(error).to.be.instanceOf(OperationStateError); + expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + }, + ); + expect(close.callCount).to.equal(1); + expect(download.callCount).to.equal(1); + } finally { + download.restore(); + } + }); + + it('keeps the operation usable after a progress callback throws', async () => { + const { context, operation } = setup(); + context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; + const close = sinon.spy(context.driver, 'closeOperation'); + const callbackError = new Error('user callback failed'); + + await operation + .fetchChunk({ + callback: () => { + throw callbackError; + }, + }) + .then( + () => expect.fail('callback should fail'), + (error) => expect(error).to.equal(callbackError), + ); + expect(close.called).to.be.false; + context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; + expect(await operation.fetchChunk({ maxRows: 3 })).to.have.length(3); + }); +}); From 13191021e601c8ff393d305a92145cda6e751773 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 09:05:32 +0000 Subject: [PATCH 2/5] fix(thrift): make failed-fetch cleanup non-blocking Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/result/RowSetProvider.ts | 17 +- lib/thrift-backend/ThriftOperationBackend.ts | 27 +++- .../unit/thrift-backend/fetchFailure.test.ts | 146 ++++++++++++++++-- 4 files changed, 176 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a664b6..5e3e85df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- After an unrecoverable Thrift result fetch or CloudFetch download error, the operation is failed and best-effort closed. Later reads throw `OperationStateError` instead of silently returning an incomplete suffix; invalid fetch arguments do not fail the operation. Kernel behavior is unchanged. +- After an unrecoverable Thrift result fetch or CloudFetch download error, later reads throw `OperationStateError` instead of silently returning an incomplete suffix. Best-effort cleanup starts without delaying the original error, logs failures, and is reused by later `close()` calls; argument handling and kernel behavior are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 8c67a9f4..9cf23e29 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -17,6 +17,12 @@ export enum FetchType { Logs = 1, } +const fetchArgumentErrors = new WeakSet(); + +export function isFetchArgumentError(error: unknown): boolean { + return error instanceof Error && fetchArgumentErrors.has(error); +} + /** * Rough byte count for a single `TColumn`. Used only for telemetry — exact * wire size isn't worth a deserialization pass. Fixed-width values are @@ -150,10 +156,19 @@ export default class RowSetProvider implements IResultsProvider; + constructor({ handle, directResults, @@ -171,9 +173,6 @@ export default class ThriftOperationBackend implements IOperationBackend { isClosed?: () => boolean; }): Promise> { this.failIfFetchFailed(); - if (Math.abs(limit) > 2 ** 64) { - throw new RangeError('maxRows is outside Int64 range'); - } const resultHandler = await this.getResultHandler(); // All the library code is Promise-based, however, since Promises are microtasks, @@ -209,8 +208,23 @@ export default class ThriftOperationBackend implements IOperationBackend { // Native GetFunctions leaves FUNCTION_CAT empty; match JDBC and kernel. return rows.map((row) => (row === null ? row : { ...row, FUNCTION_CAT: this.functionCatalog })); } catch (error) { + if (isFetchArgumentError(error)) { + throw error; + } this.fetchFailed = true; - await this.close().catch(() => undefined); + if (!this.fetchFailureClose) { + this.fetchFailureClose = this.close(); + this.fetchFailureClose.catch((closeError) => { + const cause = typeof closeError?.message === 'string' ? closeError.message : String(closeError); + this.context + .getLogger() + .log( + LogLevel.warn, + `Thrift fetch-error cleanup: close() failed for operation ${this.id}; the server-side ` + + `statement may leak until the session is closed. Cause: ${cause}`, + ); + }); + } throw error; } } @@ -374,6 +388,9 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async close(): Promise { + if (this.fetchFailureClose) { + return this.fetchFailureClose; + } this.context.getLogger().log(LogLevel.debug, `Closing operation with id: ${this.id}`); const driver = await this.context.getDriver(); const response = diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts index 564d7509..ba49fc2d 100644 --- a/tests/unit/thrift-backend/fetchFailure.test.ts +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -1,13 +1,24 @@ import { expect } from 'chai'; +import { Table, tableFromArrays, tableToIPC } from 'apache-arrow'; +import Int64 from 'node-int64'; import sinon from 'sinon'; -import { TOperationState, TOperationType, TSparkRowSetType } from '../../../thrift/TCLIService_types'; +import { + TCloseOperationResp, + TOperationState, + TOperationType, + TSparkRowSetType, + TStatusCode, + TTypeId, +} from '../../../thrift/TCLIService_types'; import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; +import StatusError from '../../../lib/errors/StatusError'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import ClientContextStub from '../.stubs/ClientContextStub'; import { createOperationForTest } from '../.stubs/createOperationForTest'; describe('Thrift fetch failure', () => { - function setup() { + function setup(directResults = false) { const context = new ClientContextStub(); context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; const operation = createOperationForTest({ @@ -17,6 +28,14 @@ describe('Thrift fetch failure', () => { operationType: TOperationType.EXECUTE_STATEMENT, hasResultSet: true, }, + directResults: directResults + ? { + operationStatus: context.driver.getOperationStatusResp, + resultSetMetadata: context.driver.getResultSetMetadataResp, + resultSet: context.driver.fetchResultsResp, + closeOperation: context.driver.closeOperationResp, + } + : undefined, }); return { context, operation }; } @@ -55,6 +74,60 @@ describe('Thrift fetch failure', () => { expect(fetch.callCount).to.equal(1); expect(close.callCount).to.equal(1); }); + + it(`reports the fetch error without waiting for cleanup and reuses its ${ + closeFails ? 'failure' : 'success' + } on close`, async () => { + const clock = sinon.useFakeTimers(); + const cleanup = sinon.promise(); + const { context, operation } = setup(); + const fetchError = new Error('fetch failed'); + const cleanupError = new StatusError({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'cleanup failed' }); + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').returns(cleanup); + const log = sinon.spy(context.logger, 'log'); + let rejectedWith: unknown; + try { + const fetching = operation.fetchChunk().catch((error) => { + rejectedWith = error; + }); + await clock.runAllAsync(); + expect(rejectedWith).to.equal(fetchError); + await fetching; + expect(close.callCount).to.equal(1); + + let closed = false; + const closing = operation.close().then((status) => { + closed = true; + return status; + }); + await clock.runAllAsync(); + expect(closed).to.be.false; + expect(close.callCount).to.equal(1); + + if (closeFails) { + cleanup.reject(cleanupError); + } else { + cleanup.resolve(context.driver.closeOperationResp); + } + const status = await closing; + expect(status.isSuccess).to.be.true; + expect(status.info).to.deep.equal(closeFails ? ['Failed to close operation: cleanup failed'] : []); + expect( + log.calledWith( + LogLevel.warn, + sinon.match('fetch-error cleanup').and(sinon.match(operation.id)).and(sinon.match('cleanup failed')), + ), + ).to.equal(closeFails); + await operation.close(); + expect(close.callCount).to.equal(1); + } finally { + if (cleanup.status === 'pending') { + cleanup.resolve(context.driver.closeOperationResp); + } + clock.restore(); + } + }); } it('keeps the operation usable after an invalid maxRows', async () => { @@ -71,16 +144,71 @@ describe('Thrift fetch failure', () => { expect(await operation.fetchChunk({ maxRows: 3 })).to.deep.equal([{ test: 'a' }, { test: 'b' }, { test: 'c' }]); }); - it('does not consume buffered rows when maxRows is invalid', async () => { + for (const maxRows of [Infinity, Number.MAX_VALUE]) { + for (const method of ['fetchChunk', 'fetchAll'] as const) { + it(`preserves direct-result ${method} with maxRows=${maxRows}`, async () => { + const { context, operation } = setup(true); + const fetch = sinon.spy(context.driver, 'fetchResults'); + expect(await operation[method]({ maxRows })).to.deep.equal([{ test: 'a' }, { test: 'b' }, { test: 'c' }]); + expect(fetch.called).to.be.false; + }); + } + } + + it('fails after consuming an Arrow batch when trailing prefetch rejects', async () => { const { context, operation } = setup(); + const table = new Table([ + ...tableFromArrays({ id: new Int32Array([0, 1]) }).batches, + ...tableFromArrays({ id: new Int32Array([2, 3]) }).batches, + ]); + const ipc = Buffer.from(tableToIPC(table, 'stream')); + const schemaLength = 8 + ipc.readInt32LE(4); + context.driver.getResultSetMetadataResp.resultFormat = TSparkRowSetType.ARROW_BASED_SET; + context.driver.getResultSetMetadataResp.arrowSchema = ipc.subarray(0, schemaLength); + context.driver.getResultSetMetadataResp.schema = { + columns: [ + { + columnName: 'id', + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] }, + position: 1, + comment: '', + }, + ], + }; + const fetchError = new Error('trailing prefetch failed'); + const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ + status: context.driver.fetchResultsResp.status, + hasMoreRows: false, + }); + fetch.onFirstCall().resolves({ + status: context.driver.fetchResultsResp.status, + hasMoreRows: true, + results: { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: ipc.subarray(schemaLength), rowCount: new Int64(4) }], + }, + }); + fetch.onSecondCall().rejects(fetchError); const close = sinon.spy(context.driver, 'closeOperation'); - expect(await operation.fetchChunk({ maxRows: 1 })).to.deep.equal([{ test: 'a' }]); - await operation.fetchChunk({ maxRows: Number.MAX_VALUE }).then( - () => expect.fail('maxRows should be outside the Int64 range'), - (error) => expect(error).to.be.instanceOf(RangeError), + + expect(await operation.fetchChunk({ maxRows: 2 })).to.deep.equal([{ id: 0 }, { id: 1 }]); + expect(fetch.callCount).to.equal(1); + await operation.fetchChunk({ maxRows: 2 }).then( + () => expect.fail('trailing prefetch should fail'), + (error) => expect(error).to.equal(fetchError), ); - expect(close.called).to.be.false; - expect(await operation.fetchChunk({ maxRows: 2 })).to.deep.equal([{ test: 'b' }, { test: 'c' }]); + for (const read of [() => operation.fetchChunk(), () => operation.fetchAll(), () => operation.hasMoreRows()]) { + await read().then( + () => expect.fail('later reads must not resume the consumed Arrow cursor'), + (error) => { + expect(error).to.be.instanceOf(OperationStateError); + expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + }, + ); + } + expect(fetch.callCount).to.equal(2); + expect(close.callCount).to.equal(1); }); it('fails after a CloudFetch provider error during fetchAll', async () => { From e68362412ab4560e53ec23eb5f0f81f99475f393 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 10:02:34 +0000 Subject: [PATCH 3/5] fix(thrift): contain cleanup errors and share close requests Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/thrift-backend/ThriftOperationBackend.ts | 42 +++++---- .../unit/thrift-backend/fetchFailure.test.ts | 89 +++++++++++++++++++ 3 files changed, 116 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3e85df..63470823 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- After an unrecoverable Thrift result fetch or CloudFetch download error, later reads throw `OperationStateError` instead of silently returning an incomplete suffix. Best-effort cleanup starts without delaying the original error, logs failures, and is reused by later `close()` calls; argument handling and kernel behavior are unchanged. +- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads throw `OperationStateError`, and callers must re-execute the statement (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares its request with `close()`; argument handling and kernel behavior are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index ae2ed668..bbbf2476 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -107,7 +107,7 @@ export default class ThriftOperationBackend implements IOperationBackend { private fetchFailed = false; - private fetchFailureClose?: Promise; + private closePromise?: Promise; constructor({ handle, @@ -211,19 +211,20 @@ export default class ThriftOperationBackend implements IOperationBackend { if (isFetchArgumentError(error)) { throw error; } - this.fetchFailed = true; - if (!this.fetchFailureClose) { - this.fetchFailureClose = this.close(); - this.fetchFailureClose.catch((closeError) => { - const cause = typeof closeError?.message === 'string' ? closeError.message : String(closeError); - this.context - .getLogger() - .log( - LogLevel.warn, - `Thrift fetch-error cleanup: close() failed for operation ${this.id}; the server-side ` + - `statement may leak until the session is closed. Cause: ${cause}`, - ); - }); + if (!this.fetchFailed) { + this.fetchFailed = true; + this.close() + .catch((closeError) => { + const cause = typeof closeError?.message === 'string' ? closeError.message : String(closeError); + this.context + .getLogger() + .log( + LogLevel.warn, + `Thrift fetch-error cleanup: close() failed for operation ${this.id}; the server-side ` + + `statement may leak until the session is closed. Cause: ${cause}`, + ); + }) + .catch(() => undefined); } throw error; } @@ -388,9 +389,18 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async close(): Promise { - if (this.fetchFailureClose) { - return this.fetchFailureClose; + if (!this.closePromise) { + this.closePromise = this.closeInternal().catch((error) => { + if (!this.fetchFailed && !this.isFinished()) { + this.closePromise = undefined; + } + throw error; + }); } + return this.closePromise; + } + + private async closeInternal(): Promise { this.context.getLogger().log(LogLevel.debug, `Closing operation with id: ${this.id}`); const driver = await this.context.getDriver(); const response = diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts index ba49fc2d..619848a2 100644 --- a/tests/unit/thrift-backend/fetchFailure.test.ts +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -4,6 +4,7 @@ import Int64 from 'node-int64'; import sinon from 'sinon'; import { TCloseOperationResp, + TFetchResultsResp, TOperationState, TOperationType, TSparkRowSetType, @@ -130,6 +131,94 @@ describe('Thrift fetch failure', () => { }); } + it('contains a throwing logger after background cleanup fails', async () => { + const { context, operation } = setup(); + const fetchError = new Error('fetch failed'); + const loggerError = new Error('logger sink closed'); + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => { + unhandled.push(error); + }; + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + sinon.stub(context.driver, 'closeOperation').rejects(new Error('cleanup failed')); + const log = sinon.stub(context.logger, 'log').callsFake((level) => { + if (level === LogLevel.warn) { + throw loggerError; + } + }); + process.on('unhandledRejection', onUnhandled); + try { + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(log.calledWith(LogLevel.warn, sinon.match('fetch-error cleanup'))).to.be.true; + expect(unhandled).to.deep.equal([]); + await operation.fetchChunk().then( + () => expect.fail('later reads must remain terminal'), + (error) => expect(error).to.be.instanceOf(OperationStateError), + ); + } finally { + process.removeListener('unhandledRejection', onUnhandled); + } + }); + + it('reuses an explicit close already in flight when a fetch fails', async () => { + const clock = sinon.useFakeTimers(); + const { context, operation } = setup(); + const fetchError = new Error('operation is closing'); + const pendingFetch = sinon.promise(); + const pendingClose = sinon.promise(); + const fetch = sinon.stub(context.driver, 'fetchResults').returns(pendingFetch); + const close = sinon.stub(context.driver, 'closeOperation').returns(pendingClose); + close.onSecondCall().rejects(new Error('operation already closed')); + const log = sinon.spy(context.logger, 'log'); + let rejectedWith: unknown; + try { + const fetching = operation.fetchChunk().catch((error) => { + rejectedWith = error; + }); + await clock.runAllAsync(); + expect(fetch.callCount).to.equal(1); + const closing = operation.close(); + await clock.runAllAsync(); + expect(close.callCount).to.equal(1); + + pendingFetch.reject(fetchError); + await clock.runAllAsync(); + expect(rejectedWith).to.equal(fetchError); + await fetching; + expect(close.callCount).to.equal(1); + pendingClose.resolve(context.driver.closeOperationResp); + expect((await closing).isSuccess).to.be.true; + expect(log.calledWith(LogLevel.warn)).to.be.false; + } finally { + if (pendingFetch.status === 'pending') { + pendingFetch.resolve(context.driver.fetchResultsResp); + } + if (pendingClose.status === 'pending') { + pendingClose.resolve(context.driver.closeOperationResp); + } + clock.restore(); + } + }); + + it('allows an unfinished explicit close to be retried after failure', async () => { + const { context, operation } = setup(); + context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; + const closeError = new Error('close failed'); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + close.onFirstCall().rejects(closeError); + await operation.close().then( + () => expect.fail('unfinished close failure should reject'), + (error) => expect(error).to.equal(closeError), + ); + expect((await operation.close()).isSuccess).to.be.true; + expect(close.callCount).to.equal(2); + }); + it('keeps the operation usable after an invalid maxRows', async () => { const { context, operation } = setup(); const close = sinon.spy(context.driver, 'closeOperation'); From 68d3f73cdd1ea8e554bc013daf923ef3a5f318d5 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 10:13:55 +0000 Subject: [PATCH 4/5] fix(thrift): preserve the original failed-fetch error Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/thrift-backend/ThriftOperationBackend.ts | 5 ++++- .../unit/thrift-backend/fetchFailure.test.ts | 22 ++++++++++--------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63470823..258b1513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads throw `OperationStateError`, and callers must re-execute the statement (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares its request with `close()`; argument handling and kernel behavior are unchanged. +- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads rethrow the original fetch error, and callers must re-execute the statement (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares its request with `close()`; argument handling and kernel behavior are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index bbbf2476..2a8fe04e 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -107,6 +107,8 @@ export default class ThriftOperationBackend implements IOperationBackend { private fetchFailed = false; + private fetchError?: unknown; + private closePromise?: Promise; constructor({ @@ -212,6 +214,7 @@ export default class ThriftOperationBackend implements IOperationBackend { throw error; } if (!this.fetchFailed) { + this.fetchError = error; this.fetchFailed = true; this.close() .catch((closeError) => { @@ -238,7 +241,7 @@ export default class ThriftOperationBackend implements IOperationBackend { private failIfFetchFailed(): void { if (this.fetchFailed) { - throw new OperationStateError(OperationStateErrorCode.Error); + throw this.fetchError; } } diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts index 619848a2..3d85dc95 100644 --- a/tests/unit/thrift-backend/fetchFailure.test.ts +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -11,7 +11,7 @@ import { TStatusCode, TTypeId, } from '../../../thrift/TCLIService_types'; -import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; import StatusError from '../../../lib/errors/StatusError'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; @@ -46,7 +46,7 @@ describe('Thrift fetch failure', () => { closeFails ? 'fails' : 'succeeds' }`, async () => { const { context, operation } = setup(); - const fetchError = new Error('exhausted fetch retries'); + const fetchError = new HiveDriverError('exhausted fetch retries'); const fetch = sinon.stub(context.driver, 'fetchResults').callThrough(); fetch.onFirstCall().rejects(fetchError); const close = sinon.stub(context.driver, 'closeOperation').callThrough(); @@ -67,8 +67,8 @@ describe('Thrift fetch failure', () => { await read().then( () => expect.fail('later reads must not recover a suffix'), (error) => { - expect(error).to.be.instanceOf(OperationStateError); - expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + expect(error).to.equal(fetchError); + expect(error).to.be.instanceOf(HiveDriverError); }, ); } @@ -96,6 +96,10 @@ describe('Thrift fetch failure', () => { expect(rejectedWith).to.equal(fetchError); await fetching; expect(close.callCount).to.equal(1); + await operation.hasMoreRows().then( + () => expect.fail('later reads must fail while cleanup is pending'), + (error) => expect(error).to.equal(fetchError), + ); let closed = false; const closing = operation.close().then((status) => { @@ -158,7 +162,7 @@ describe('Thrift fetch failure', () => { expect(unhandled).to.deep.equal([]); await operation.fetchChunk().then( () => expect.fail('later reads must remain terminal'), - (error) => expect(error).to.be.instanceOf(OperationStateError), + (error) => expect(error).to.equal(fetchError), ); } finally { process.removeListener('unhandledRejection', onUnhandled); @@ -264,7 +268,7 @@ describe('Thrift fetch failure', () => { }, ], }; - const fetchError = new Error('trailing prefetch failed'); + const fetchError = new HiveDriverError('trailing prefetch failed'); const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ status: context.driver.fetchResultsResp.status, hasMoreRows: false, @@ -291,8 +295,7 @@ describe('Thrift fetch failure', () => { await read().then( () => expect.fail('later reads must not resume the consumed Arrow cursor'), (error) => { - expect(error).to.be.instanceOf(OperationStateError); - expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + expect(error).to.equal(fetchError); }, ); } @@ -314,8 +317,7 @@ describe('Thrift fetch failure', () => { await operation.fetchChunk().then( () => expect.fail('later reads must fail'), (error) => { - expect(error).to.be.instanceOf(OperationStateError); - expect(error.errorCode).to.equal(OperationStateErrorCode.Error); + expect(error).to.equal(downloadError); }, ); expect(close.callCount).to.equal(1); From d8c17ec2596f651d9cce0b9897298a1095faf71d Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 16:15:36 +0000 Subject: [PATCH 5/5] fix(thrift): yield failed reads and retry settled cleanup Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/DBSQLOperation.ts | 4 ++ lib/thrift-backend/ThriftOperationBackend.ts | 14 ++-- .../unit/thrift-backend/fetchFailure.test.ts | 64 ++++++++++++++++++- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 258b1513..2d90a9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads rethrow the original fetch error, and callers must re-execute the statement (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares its request with `close()`; argument handling and kernel behavior are unchanged. +- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads replay the same error object, so callers re-execute rather than retry the read (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares only pending close requests so a later `close()` can retry failed cleanup; argument handling and kernel behavior are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 30f095ef..b6e4e842 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -110,6 +110,8 @@ export default class DBSQLOperation implements IOperation { /** * Fetches all data + * A failed Thrift result fetch is terminal; later reads replay the same error object. + * Re-execute the statement instead of retrying the read. * @public * @param options - maxRows property can be set to limit chunk size * @returns Array of data with length equal to option.maxRows @@ -137,6 +139,8 @@ export default class DBSQLOperation implements IOperation { /** * Fetches chunk of data + * A failed Thrift result fetch is terminal; later reads replay the same error object. + * Re-execute the statement instead of retrying the read. * @public * @param options - maxRows property sets chunk size * @returns Array of data with length equal to option.maxRows diff --git a/lib/thrift-backend/ThriftOperationBackend.ts b/lib/thrift-backend/ThriftOperationBackend.ts index 2a8fe04e..b1b93f40 100644 --- a/lib/thrift-backend/ThriftOperationBackend.ts +++ b/lib/thrift-backend/ThriftOperationBackend.ts @@ -174,7 +174,6 @@ export default class ThriftOperationBackend implements IOperationBackend { disableBuffering?: boolean; isClosed?: () => boolean; }): Promise> { - this.failIfFetchFailed(); const resultHandler = await this.getResultHandler(); // All the library code is Promise-based, however, since Promises are microtasks, @@ -190,6 +189,7 @@ export default class ThriftOperationBackend implements IOperationBackend { await new Promise((resolve) => { setTimeout(resolve, 0); }); + this.failIfFetchFailed(); // Mid-flight cancel/close probe. The pre-refactor facade had a // `failIfClosed()` between metadata-prep and the data RPC; that guard moved @@ -234,7 +234,10 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async hasMore(): Promise { - this.failIfFetchFailed(); + if (this.fetchFailed) { + await delay(0); + this.failIfFetchFailed(); + } const resultHandler = await this.getResultHandler(); return resultHandler.hasMore(); } @@ -393,11 +396,8 @@ export default class ThriftOperationBackend implements IOperationBackend { public async close(): Promise { if (!this.closePromise) { - this.closePromise = this.closeInternal().catch((error) => { - if (!this.fetchFailed && !this.isFinished()) { - this.closePromise = undefined; - } - throw error; + this.closePromise = this.closeInternal().finally(() => { + this.closePromise = undefined; }); } return this.closePromise; diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts index 3d85dc95..0905a4d5 100644 --- a/tests/unit/thrift-backend/fetchFailure.test.ts +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -76,7 +76,7 @@ describe('Thrift fetch failure', () => { expect(close.callCount).to.equal(1); }); - it(`reports the fetch error without waiting for cleanup and reuses its ${ + it(`reports the fetch error without waiting and shares pending cleanup's ${ closeFails ? 'failure' : 'success' } on close`, async () => { const clock = sinon.useFakeTimers(); @@ -96,10 +96,12 @@ describe('Thrift fetch failure', () => { expect(rejectedWith).to.equal(fetchError); await fetching; expect(close.callCount).to.equal(1); - await operation.hasMoreRows().then( + const laterRead = operation.hasMoreRows().then( () => expect.fail('later reads must fail while cleanup is pending'), (error) => expect(error).to.equal(fetchError), ); + await clock.runAllAsync(); + await laterRead; let closed = false; const closing = operation.close().then((status) => { @@ -135,6 +137,64 @@ describe('Thrift fetch failure', () => { }); } + it('yields to a zero-delay timer between latched read rejections', async () => { + const { context, operation } = setup(); + const fetchError = new HiveDriverError('transient fetch failed'); + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + + for (const method of ['fetchChunk', 'fetchAll', 'hasMoreRows'] as const) { + await operation[method]().then( + () => expect.fail('later reads must fail'), + (error) => expect(error).to.equal(fetchError), + ); + let timerFired = false; + const timer = new Promise((resolve) => { + setTimeout(() => { + timerFired = true; + resolve(); + }, 0); + }); + try { + await operation[method]().then( + () => expect.fail('later reads must fail'), + (error) => expect(error).to.equal(fetchError), + ); + expect(timerFired, `${method} must not starve macrotasks`).to.be.true; + } finally { + await timer; + } + } + expect(fetch.callCount).to.equal(1); + }); + + it('makes a fresh explicit close attempt after background cleanup fails', async () => { + const { context, operation } = setup(); + const fetchError = new HiveDriverError('fetch failed'); + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + close.onFirstCall().rejects(new Error('cleanup failed during outage')); + const log = sinon.spy(context.logger, 'log'); + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(close.callCount).to.equal(1); + expect(log.calledWith(LogLevel.warn, sinon.match('fetch-error cleanup'))).to.be.true; + + const status = await operation.close(); + expect(close.callCount).to.equal(2); + expect(status.isSuccess).to.be.true; + expect(status.info).to.deep.equal([]); + expect(log.calledWith(LogLevel.warn, sinon.match('Failed to close finished operation'))).to.be.false; + await operation.close(); + expect(close.callCount).to.equal(2); + }); + it('contains a throwing logger after background cleanup fails', async () => { const { context, operation } = setup(); const fetchError = new Error('fetch failed');