diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..2d90a9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Any failed Thrift `FetchResults` call or CloudFetch download makes the operation terminal: later reads replay the same error object, so callers re-execute rather than retry the read (`FetchResults` is not retried). Cleanup runs in the background under the existing close retry budget, logs failures safely, and shares only pending close requests so a later `close()` can retry failed cleanup; argument handling and kernel behavior are unchanged. + - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/DBSQLOperation.ts b/lib/DBSQLOperation.ts index 30f095ef..b6e4e842 100644 --- a/lib/DBSQLOperation.ts +++ b/lib/DBSQLOperation.ts @@ -110,6 +110,8 @@ export default class DBSQLOperation implements IOperation { /** * Fetches all data + * A failed Thrift result fetch is terminal; later reads replay the same error object. + * Re-execute the statement instead of retrying the read. * @public * @param options - maxRows property can be set to limit chunk size * @returns Array of data with length equal to option.maxRows @@ -137,6 +139,8 @@ export default class DBSQLOperation implements IOperation { /** * Fetches chunk of data + * A failed Thrift result fetch is terminal; later reads replay the same error object. + * Re-execute the statement instead of retrying the read. * @public * @param options - maxRows property sets chunk size * @returns Array of data with length equal to option.maxRows diff --git a/lib/result/RowSetProvider.ts b/lib/result/RowSetProvider.ts index 8c67a9f4..9cf23e29 100644 --- a/lib/result/RowSetProvider.ts +++ b/lib/result/RowSetProvider.ts @@ -17,6 +17,12 @@ export enum FetchType { Logs = 1, } +const fetchArgumentErrors = new WeakSet(); + +export function isFetchArgumentError(error: unknown): boolean { + return error instanceof Error && fetchArgumentErrors.has(error); +} + /** * Rough byte count for a single `TColumn`. Used only for telemetry — exact * wire size isn't worth a deserialization pass. Fixed-width values are @@ -150,10 +156,19 @@ export default class RowSetProvider implements IResultsProvider; + private fetchFailed = false; + + private fetchError?: unknown; + + private closePromise?: Promise; + constructor({ handle, directResults, @@ -183,6 +189,7 @@ export default class ThriftOperationBackend implements IOperationBackend { await new Promise((resolve) => { setTimeout(resolve, 0); }); + this.failIfFetchFailed(); // Mid-flight cancel/close probe. The pre-refactor facade had a // `failIfClosed()` between metadata-prep and the data RPC; that guard moved @@ -194,20 +201,53 @@ export default class ThriftOperationBackend implements IOperationBackend { return []; } - const rows = await resultHandler.fetchNext({ limit, disableBuffering }); - if (this.functionCatalog === undefined) { - return rows; - } + try { + const rows = await resultHandler.fetchNext({ limit, disableBuffering }); + if (this.functionCatalog === undefined) { + return rows; + } - // Native GetFunctions leaves FUNCTION_CAT empty; match JDBC and kernel. - return rows.map((row) => (row === null ? row : { ...row, FUNCTION_CAT: this.functionCatalog })); + // Native GetFunctions leaves FUNCTION_CAT empty; match JDBC and kernel. + return rows.map((row) => (row === null ? row : { ...row, FUNCTION_CAT: this.functionCatalog })); + } catch (error) { + if (isFetchArgumentError(error)) { + throw error; + } + if (!this.fetchFailed) { + this.fetchError = error; + this.fetchFailed = true; + this.close() + .catch((closeError) => { + const cause = typeof closeError?.message === 'string' ? closeError.message : String(closeError); + this.context + .getLogger() + .log( + LogLevel.warn, + `Thrift fetch-error cleanup: close() failed for operation ${this.id}; the server-side ` + + `statement may leak until the session is closed. Cause: ${cause}`, + ); + }) + .catch(() => undefined); + } + throw error; + } } public async hasMore(): Promise { + if (this.fetchFailed) { + await delay(0); + this.failIfFetchFailed(); + } const resultHandler = await this.getResultHandler(); return resultHandler.hasMore(); } + private failIfFetchFailed(): void { + if (this.fetchFailed) { + throw this.fetchError; + } + } + public async status(progress: boolean): Promise { const response = await this.thriftStatusResponse(progress); return this.adaptOperationStatus(response); @@ -355,6 +395,15 @@ export default class ThriftOperationBackend implements IOperationBackend { } public async close(): Promise { + if (!this.closePromise) { + this.closePromise = this.closeInternal().finally(() => { + this.closePromise = undefined; + }); + } + return this.closePromise; + } + + private async closeInternal(): Promise { this.context.getLogger().log(LogLevel.debug, `Closing operation with id: ${this.id}`); const driver = await this.context.getDriver(); const response = diff --git a/tests/unit/thrift-backend/fetchFailure.test.ts b/tests/unit/thrift-backend/fetchFailure.test.ts new file mode 100644 index 00000000..0905a4d5 --- /dev/null +++ b/tests/unit/thrift-backend/fetchFailure.test.ts @@ -0,0 +1,410 @@ +import { expect } from 'chai'; +import { Table, tableFromArrays, tableToIPC } from 'apache-arrow'; +import Int64 from 'node-int64'; +import sinon from 'sinon'; +import { + TCloseOperationResp, + TFetchResultsResp, + TOperationState, + TOperationType, + TSparkRowSetType, + TStatusCode, + TTypeId, +} from '../../../thrift/TCLIService_types'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; +import StatusError from '../../../lib/errors/StatusError'; +import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import { createOperationForTest } from '../.stubs/createOperationForTest'; + +describe('Thrift fetch failure', () => { + function setup(directResults = false) { + const context = new ClientContextStub(); + context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; + const operation = createOperationForTest({ + context, + handle: { + operationId: { guid: Buffer.alloc(16), secret: Buffer.alloc(16) }, + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: true, + }, + directResults: directResults + ? { + operationStatus: context.driver.getOperationStatusResp, + resultSetMetadata: context.driver.getResultSetMetadataResp, + resultSet: context.driver.fetchResultsResp, + closeOperation: context.driver.closeOperationResp, + } + : undefined, + }); + return { context, operation }; + } + + for (const closeFails of [false, true]) { + it(`rejects later reads after an exhausted fetch, even when close ${ + closeFails ? 'fails' : 'succeeds' + }`, async () => { + const { context, operation } = setup(); + const fetchError = new HiveDriverError('exhausted fetch retries'); + const fetch = sinon.stub(context.driver, 'fetchResults').callThrough(); + fetch.onFirstCall().rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + if (closeFails) { + close.rejects(new Error('close failed')); + } + + await operation.fetchChunk({ maxRows: 3 }).then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + + for (const read of [ + () => operation.fetchChunk({ maxRows: 3 }), + () => operation.fetchAll(), + () => operation.hasMoreRows(), + ]) { + await read().then( + () => expect.fail('later reads must not recover a suffix'), + (error) => { + expect(error).to.equal(fetchError); + expect(error).to.be.instanceOf(HiveDriverError); + }, + ); + } + expect(fetch.callCount).to.equal(1); + expect(close.callCount).to.equal(1); + }); + + it(`reports the fetch error without waiting and shares pending cleanup's ${ + closeFails ? 'failure' : 'success' + } on close`, async () => { + const clock = sinon.useFakeTimers(); + const cleanup = sinon.promise(); + const { context, operation } = setup(); + const fetchError = new Error('fetch failed'); + const cleanupError = new StatusError({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'cleanup failed' }); + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').returns(cleanup); + const log = sinon.spy(context.logger, 'log'); + let rejectedWith: unknown; + try { + const fetching = operation.fetchChunk().catch((error) => { + rejectedWith = error; + }); + await clock.runAllAsync(); + expect(rejectedWith).to.equal(fetchError); + await fetching; + expect(close.callCount).to.equal(1); + const laterRead = operation.hasMoreRows().then( + () => expect.fail('later reads must fail while cleanup is pending'), + (error) => expect(error).to.equal(fetchError), + ); + await clock.runAllAsync(); + await laterRead; + + let closed = false; + const closing = operation.close().then((status) => { + closed = true; + return status; + }); + await clock.runAllAsync(); + expect(closed).to.be.false; + expect(close.callCount).to.equal(1); + + if (closeFails) { + cleanup.reject(cleanupError); + } else { + cleanup.resolve(context.driver.closeOperationResp); + } + const status = await closing; + expect(status.isSuccess).to.be.true; + expect(status.info).to.deep.equal(closeFails ? ['Failed to close operation: cleanup failed'] : []); + expect( + log.calledWith( + LogLevel.warn, + sinon.match('fetch-error cleanup').and(sinon.match(operation.id)).and(sinon.match('cleanup failed')), + ), + ).to.equal(closeFails); + await operation.close(); + expect(close.callCount).to.equal(1); + } finally { + if (cleanup.status === 'pending') { + cleanup.resolve(context.driver.closeOperationResp); + } + clock.restore(); + } + }); + } + + it('yields to a zero-delay timer between latched read rejections', async () => { + const { context, operation } = setup(); + const fetchError = new HiveDriverError('transient fetch failed'); + const fetch = sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + + for (const method of ['fetchChunk', 'fetchAll', 'hasMoreRows'] as const) { + await operation[method]().then( + () => expect.fail('later reads must fail'), + (error) => expect(error).to.equal(fetchError), + ); + let timerFired = false; + const timer = new Promise((resolve) => { + setTimeout(() => { + timerFired = true; + resolve(); + }, 0); + }); + try { + await operation[method]().then( + () => expect.fail('later reads must fail'), + (error) => expect(error).to.equal(fetchError), + ); + expect(timerFired, `${method} must not starve macrotasks`).to.be.true; + } finally { + await timer; + } + } + expect(fetch.callCount).to.equal(1); + }); + + it('makes a fresh explicit close attempt after background cleanup fails', async () => { + const { context, operation } = setup(); + const fetchError = new HiveDriverError('fetch failed'); + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + close.onFirstCall().rejects(new Error('cleanup failed during outage')); + const log = sinon.spy(context.logger, 'log'); + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(close.callCount).to.equal(1); + expect(log.calledWith(LogLevel.warn, sinon.match('fetch-error cleanup'))).to.be.true; + + const status = await operation.close(); + expect(close.callCount).to.equal(2); + expect(status.isSuccess).to.be.true; + expect(status.info).to.deep.equal([]); + expect(log.calledWith(LogLevel.warn, sinon.match('Failed to close finished operation'))).to.be.false; + await operation.close(); + expect(close.callCount).to.equal(2); + }); + + it('contains a throwing logger after background cleanup fails', async () => { + const { context, operation } = setup(); + const fetchError = new Error('fetch failed'); + const loggerError = new Error('logger sink closed'); + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => { + unhandled.push(error); + }; + sinon.stub(context.driver, 'fetchResults').rejects(fetchError); + sinon.stub(context.driver, 'closeOperation').rejects(new Error('cleanup failed')); + const log = sinon.stub(context.logger, 'log').callsFake((level) => { + if (level === LogLevel.warn) { + throw loggerError; + } + }); + process.on('unhandledRejection', onUnhandled); + try { + await operation.fetchChunk().then( + () => expect.fail('fetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(log.calledWith(LogLevel.warn, sinon.match('fetch-error cleanup'))).to.be.true; + expect(unhandled).to.deep.equal([]); + await operation.fetchChunk().then( + () => expect.fail('later reads must remain terminal'), + (error) => expect(error).to.equal(fetchError), + ); + } finally { + process.removeListener('unhandledRejection', onUnhandled); + } + }); + + it('reuses an explicit close already in flight when a fetch fails', async () => { + const clock = sinon.useFakeTimers(); + const { context, operation } = setup(); + const fetchError = new Error('operation is closing'); + const pendingFetch = sinon.promise(); + const pendingClose = sinon.promise(); + const fetch = sinon.stub(context.driver, 'fetchResults').returns(pendingFetch); + const close = sinon.stub(context.driver, 'closeOperation').returns(pendingClose); + close.onSecondCall().rejects(new Error('operation already closed')); + const log = sinon.spy(context.logger, 'log'); + let rejectedWith: unknown; + try { + const fetching = operation.fetchChunk().catch((error) => { + rejectedWith = error; + }); + await clock.runAllAsync(); + expect(fetch.callCount).to.equal(1); + const closing = operation.close(); + await clock.runAllAsync(); + expect(close.callCount).to.equal(1); + + pendingFetch.reject(fetchError); + await clock.runAllAsync(); + expect(rejectedWith).to.equal(fetchError); + await fetching; + expect(close.callCount).to.equal(1); + pendingClose.resolve(context.driver.closeOperationResp); + expect((await closing).isSuccess).to.be.true; + expect(log.calledWith(LogLevel.warn)).to.be.false; + } finally { + if (pendingFetch.status === 'pending') { + pendingFetch.resolve(context.driver.fetchResultsResp); + } + if (pendingClose.status === 'pending') { + pendingClose.resolve(context.driver.closeOperationResp); + } + clock.restore(); + } + }); + + it('allows an unfinished explicit close to be retried after failure', async () => { + const { context, operation } = setup(); + context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; + const closeError = new Error('close failed'); + const close = sinon.stub(context.driver, 'closeOperation').callThrough(); + close.onFirstCall().rejects(closeError); + await operation.close().then( + () => expect.fail('unfinished close failure should reject'), + (error) => expect(error).to.equal(closeError), + ); + expect((await operation.close()).isSuccess).to.be.true; + expect(close.callCount).to.equal(2); + }); + + it('keeps the operation usable after an invalid maxRows', async () => { + const { context, operation } = setup(); + const close = sinon.spy(context.driver, 'closeOperation'); + const fetch = sinon.spy(context.driver, 'fetchResults'); + + await operation.fetchChunk({ maxRows: Number.MAX_VALUE }).then( + () => expect.fail('maxRows should be outside the Int64 range'), + (error) => expect(error).to.be.instanceOf(RangeError), + ); + expect(close.called).to.be.false; + expect(fetch.called).to.be.false; + expect(await operation.fetchChunk({ maxRows: 3 })).to.deep.equal([{ test: 'a' }, { test: 'b' }, { test: 'c' }]); + }); + + for (const maxRows of [Infinity, Number.MAX_VALUE]) { + for (const method of ['fetchChunk', 'fetchAll'] as const) { + it(`preserves direct-result ${method} with maxRows=${maxRows}`, async () => { + const { context, operation } = setup(true); + const fetch = sinon.spy(context.driver, 'fetchResults'); + expect(await operation[method]({ maxRows })).to.deep.equal([{ test: 'a' }, { test: 'b' }, { test: 'c' }]); + expect(fetch.called).to.be.false; + }); + } + } + + it('fails after consuming an Arrow batch when trailing prefetch rejects', async () => { + const { context, operation } = setup(); + const table = new Table([ + ...tableFromArrays({ id: new Int32Array([0, 1]) }).batches, + ...tableFromArrays({ id: new Int32Array([2, 3]) }).batches, + ]); + const ipc = Buffer.from(tableToIPC(table, 'stream')); + const schemaLength = 8 + ipc.readInt32LE(4); + context.driver.getResultSetMetadataResp.resultFormat = TSparkRowSetType.ARROW_BASED_SET; + context.driver.getResultSetMetadataResp.arrowSchema = ipc.subarray(0, schemaLength); + context.driver.getResultSetMetadataResp.schema = { + columns: [ + { + columnName: 'id', + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] }, + position: 1, + comment: '', + }, + ], + }; + const fetchError = new HiveDriverError('trailing prefetch failed'); + const fetch = sinon.stub(context.driver, 'fetchResults').resolves({ + status: context.driver.fetchResultsResp.status, + hasMoreRows: false, + }); + fetch.onFirstCall().resolves({ + status: context.driver.fetchResultsResp.status, + hasMoreRows: true, + results: { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: ipc.subarray(schemaLength), rowCount: new Int64(4) }], + }, + }); + fetch.onSecondCall().rejects(fetchError); + const close = sinon.spy(context.driver, 'closeOperation'); + + expect(await operation.fetchChunk({ maxRows: 2 })).to.deep.equal([{ id: 0 }, { id: 1 }]); + expect(fetch.callCount).to.equal(1); + await operation.fetchChunk({ maxRows: 2 }).then( + () => expect.fail('trailing prefetch should fail'), + (error) => expect(error).to.equal(fetchError), + ); + for (const read of [() => operation.fetchChunk(), () => operation.fetchAll(), () => operation.hasMoreRows()]) { + await read().then( + () => expect.fail('later reads must not resume the consumed Arrow cursor'), + (error) => { + expect(error).to.equal(fetchError); + }, + ); + } + expect(fetch.callCount).to.equal(2); + expect(close.callCount).to.equal(1); + }); + + it('fails after a CloudFetch provider error during fetchAll', async () => { + const { context, operation } = setup(); + context.driver.getResultSetMetadataResp.resultFormat = TSparkRowSetType.URL_BASED_SET; + const close = sinon.spy(context.driver, 'closeOperation'); + const downloadError = new RangeError('invalid downloaded Arrow data'); + const download = sinon.stub(CloudFetchResultHandler.prototype, 'fetchNext').rejects(downloadError); + try { + await operation.fetchAll().then( + () => expect.fail('fetchAll should fail'), + (error) => expect(error).to.equal(downloadError), + ); + await operation.fetchChunk().then( + () => expect.fail('later reads must fail'), + (error) => { + expect(error).to.equal(downloadError); + }, + ); + expect(close.callCount).to.equal(1); + expect(download.callCount).to.equal(1); + } finally { + download.restore(); + } + }); + + it('keeps the operation usable after a progress callback throws', async () => { + const { context, operation } = setup(); + context.driver.getOperationStatusResp.operationState = TOperationState.RUNNING_STATE; + const close = sinon.spy(context.driver, 'closeOperation'); + const callbackError = new Error('user callback failed'); + + await operation + .fetchChunk({ + callback: () => { + throw callbackError; + }, + }) + .then( + () => expect.fail('callback should fail'), + (error) => expect(error).to.equal(callbackError), + ); + expect(close.called).to.be.false; + context.driver.getOperationStatusResp.operationState = TOperationState.FINISHED_STATE; + expect(await operation.fetchChunk({ maxRows: 3 })).to.have.length(3); + }); +});