diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..7359ff2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Avoid an extra full-buffer copy when downloading Thrift CloudFetch results. + - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index fbfe5c7c..71d8c44b 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -144,12 +144,12 @@ export default class CloudFetchResultHandler implements IResultsProvider { + private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: Buffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; if (typeof requestUrl !== 'string') { @@ -175,9 +175,9 @@ export default class CloudFetchResultHandler implements IResultsProvider { @@ -194,7 +194,7 @@ export default class CloudFetchResultHandler implements IResultsProvider { expect(batches).to.deep.eq([sampleArrowBatch]); }); + [Buffer.alloc(0), Buffer.from([0, 255, 128, 42]), Buffer.from([99, 0, 255, 128, 42, 98]).subarray(1, 5)].forEach( + (body, index) => { + it(`should reuse the fallback response buffer without copying (${index})`, async () => { + const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1 }); + const rowSetProvider = new ResultsProviderStub([sampleRowSet1], undefined); + const result = new CloudFetchResultHandler(context, rowSetProvider, { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + const response = new Response(body, { status: 200 }); + const buffer = sinon.spy(response, 'buffer'); + const arrayBuffer = sinon.spy(response, 'arrayBuffer'); + context.invokeWithRetryStub.resolves({ request: new Request('localhost'), response }); + + const { batches, rowCount } = await result.fetchNext({ limit: 10000 }); + + expect(batches).to.have.lengthOf(1); + expect(batches[0]).to.equal(body); + expect(batches[0].byteLength).to.equal(body.byteLength); + expect(rowCount).to.equal(1); + expect(buffer.calledOnce).to.be.true; + expect(arrayBuffer.called).to.be.false; + }); + }, + ); + describe('download deadline', () => { let server: http.Server | undefined; @@ -789,6 +816,42 @@ describe('CloudFetchResultHandler', () => { return { ...outcome, elapsedMs: Date.now() - startedAt }; } + it('should decode a pooled Arrow response read inside the retry callback', async () => { + const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]); + const url = await startServer((req, res) => res.end(body)); + const handler = createHandler(url, { retriesTimeout: 1000 }); + const buffer = sinon.spy(Response.prototype, 'buffer'); + const arrayBuffer = sinon.spy(Response.prototype, 'arrayBuffer'); + const fetchNext = sinon.spy(handler, 'fetchNext'); + const converter = new ArrowResultConverter(new BaseClientContextStub(), handler, { + schema: { + columns: [ + { + columnName: '1', + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] }, + position: 1, + }, + ], + }, + }); + + try { + expect(await converter.fetchNext({ limit: 10000 })).to.deep.eq([{ 1: 1 }]); + expect(buffer.calledOnce).to.be.true; + expect(arrayBuffer.called).to.be.false; + expect(buffer.firstCall.thisValue.body).to.have.property('pipe').that.is.a('function'); + const downloaded = await buffer.firstCall.returnValue; + expect(downloaded).to.deep.eq(body); + expect(downloaded.length).to.be.lt(Buffer.poolSize >>> 1); + expect(downloaded.buffer.byteLength).to.be.gt(downloaded.length); + expect((await fetchNext.firstCall.returnValue).batches[0]).to.equal(downloaded); + } finally { + fetchNext.restore(); + arrayBuffer.restore(); + buffer.restore(); + } + }); + it('should fail a download whose response headers never arrive with a timeout error', async () => { const closedRequests: Array> = []; const url = await startServer((req) => {