From 4a821bb00416f44eb6c8603ecdda19de1e0c0afb Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 07:27:29 +0000 Subject: [PATCH 1/3] Allow bounded background telemetry export during client close Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 + lib/DBSQLClient.ts | 17 +- lib/contracts/IDBSQLClient.ts | 13 +- lib/index.ts | 1 + lib/telemetry/DatabricksTelemetryExporter.ts | 32 +- lib/telemetry/TelemetryClient.ts | 4 +- lib/telemetry/TelemetryClientProvider.ts | 30 +- tests/e2e/telemetry/close-e2e.test.ts | 55 +++ tests/unit/telemetry/BackgroundClose.test.ts | 335 +++++++++++++++++++ 9 files changed, 469 insertions(+), 20 deletions(-) create mode 100644 tests/e2e/telemetry/close-e2e.test.ts create mode 100644 tests/unit/telemetry/BackgroundClose.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..11711a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +Add opt-in `client.close({ waitForTelemetry: false })` for bounded background Thrift telemetry export. The default still awaits export; explicit process exit or runtime suspension can lose the background batch. + - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index efc150d9..aaa000bd 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -5,7 +5,12 @@ import tls from 'tls'; import { EventEmitter } from 'events'; import TCLIService from '../thrift/TCLIService'; -import IDBSQLClient, { ClientOptions, ConnectionOptions, OpenSessionRequest } from './contracts/IDBSQLClient'; +import IDBSQLClient, { + ClientOptions, + ClientCloseOptions, + ConnectionOptions, + OpenSessionRequest, +} from './contracts/IDBSQLClient'; import IDriver from './contracts/IDriver'; import IClientContext, { ClientConfig } from './contracts/IClientContext'; import IThriftClient from './contracts/IThriftClient'; @@ -946,8 +951,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I * telemetry buffered between flush ticks is lost if the process exits * without calling `close()`. Long-lived applications should `await` this * method on shutdown so the aggregator drains its remaining metrics. + * `waitForTelemetry: false` opts into a bounded background Thrift export; + * explicit process exit or runtime suspension can lose the final batch. */ - public async close(): Promise { + public async close(options: ClientCloseOptions = {}): Promise { try { await this.sessions.closeAll(); await this.backend?.close(); @@ -957,13 +964,9 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I this.backend = undefined; - // Cleanup telemetry. Releasing our refcount on the shared TelemetryClient - // is awaited because the underlying close() drains the final HTTP POST — - // a caller doing `await client.close(); process.exit(0)` would otherwise - // truncate the in-flight request when this is the last refcount holder. if (this.host && this.telemetryClient) { try { - await TelemetryClientProvider.getInstance().releaseClient(this, this.host); + await TelemetryClientProvider.getInstance().releaseClient(this, this.host, options.waitForTelemetry); } catch (error: any) { this.logger.log(LogLevel.debug, `Telemetry cleanup error: ${error.message}`); } diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 5c302bb0..3519e499 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -10,6 +10,17 @@ export interface ClientOptions { logger?: IDBSQLLogger; } +export interface ClientCloseOptions { + /** + * Wait for the final Thrift telemetry export (default true). With false, + * export runs in the background for up to telemetryCloseTimeoutMs (2s by + * default); a full background queue falls back to waiting. process.exit() + * or a frozen runtime can lose that batch. Native kernel session cleanup, + * including its telemetry flush, is always awaited. + */ + waitForTelemetry?: boolean; +} + type AuthOptions = | { authType?: 'access-token'; @@ -319,5 +330,5 @@ export default interface IDBSQLClient { openSession(request?: OpenSessionRequest): Promise; - close(): Promise; + close(options?: ClientCloseOptions): Promise; } diff --git a/lib/index.ts b/lib/index.ts index 51e2aa8b..af9dee66 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -21,6 +21,7 @@ import { formatProgress } from './utils'; import { LogLevel } from './contracts/IDBSQLLogger'; // Re-export types for TypeScript users +export type { ClientCloseOptions } from './contracts/IDBSQLClient'; export type { default as ITokenProvider } from './connection/auth/tokenProvider/ITokenProvider'; export type { RowMode } from './contracts/IDBSQLSession'; diff --git a/lib/telemetry/DatabricksTelemetryExporter.ts b/lib/telemetry/DatabricksTelemetryExporter.ts index 8ef1b0a0..b33d6d64 100644 --- a/lib/telemetry/DatabricksTelemetryExporter.ts +++ b/lib/telemetry/DatabricksTelemetryExporter.ts @@ -19,6 +19,7 @@ import fetch, { RequestInit, Response, Request } from 'node-fetch'; import IClientContext from '../contracts/IClientContext'; import { LogLevel } from '../contracts/IDBSQLLogger'; import IAuthentication from '../connection/contracts/IAuthentication'; +import IConnectionProvider from '../connection/contracts/IConnectionProvider'; import AuthenticationError from '../errors/AuthenticationError'; import HiveDriverError from '../errors/HiveDriverError'; import { TelemetryMetric, DEFAULT_TELEMETRY_CONFIG } from './types'; @@ -125,6 +126,10 @@ export default class DatabricksTelemetryExporter { private authMissingWarned = false; + private readonly abortController = new AbortController(); + + private connectionProvider?: Promise; + constructor( private context: IClientContext, private host: string, @@ -156,10 +161,19 @@ export default class DatabricksTelemetryExporter { * the last consumer on its host; multi-consumer refcounting on the * registry will land in the consumer-wiring PR. */ - dispose(): void { + dispose(cancelPendingExports: boolean = false): void { + if (cancelPendingExports) { + this.abortController.abort(); + } this.circuitBreakerRegistry.removeCircuitBreaker(this.host); } + detachContext(): void { + this.authProvider = this.authProvider ?? this.context.getAuthProvider?.(); + this.connectionProvider = this.context.getConnectionProvider(); + this.connectionProvider.catch(() => {}); + } + async export(metrics: TelemetryMetric[]): Promise { if (!metrics || metrics.length === 0) { return; @@ -207,10 +221,12 @@ export default class DatabricksTelemetryExporter { /* eslint-disable no-await-in-loop */ for (let attempt = 0; attempt < totalAttempts; attempt += 1) { + if (this.abortController.signal.aborted) return; try { await this.exportInternal(metrics); return; } catch (error: any) { + if (this.abortController.signal.aborted) return; lastError = error; if ( @@ -299,6 +315,7 @@ export default class DatabricksTelemetryExporter { headers, body, timeout: 10000, + signal: this.abortController.signal as unknown as RequestInit['signal'], }); if (!response.ok) { @@ -333,7 +350,7 @@ export default class DatabricksTelemetryExporter { } private async sendRequest(url: string, init: RequestInit): Promise { - const connectionProvider = await this.context.getConnectionProvider(); + const connectionProvider = await (this.connectionProvider ?? this.context.getConnectionProvider()); const agent = await connectionProvider.getAgent(url); // Do NOT route through `connectionProvider.getRetryPolicy()` here. That // policy is bounded by the customer's `retriesTimeout` (15 min default) @@ -472,7 +489,16 @@ export default class DatabricksTelemetryExporter { private sleep(ms: number): Promise { return new Promise((resolve) => { - setTimeout(resolve, ms); + const { signal } = this.abortController; + let timer: NodeJS.Timeout; + const finish = (): void => { + clearTimeout(timer); + signal.removeEventListener('abort', finish); + resolve(); + }; + timer = setTimeout(finish, ms); + if (signal.aborted) finish(); + else signal.addEventListener('abort', finish, { once: true }); }); } } diff --git a/lib/telemetry/TelemetryClient.ts b/lib/telemetry/TelemetryClient.ts index c8f55e95..a2dc3fc8 100644 --- a/lib/telemetry/TelemetryClient.ts +++ b/lib/telemetry/TelemetryClient.ts @@ -306,7 +306,7 @@ class TelemetryClient implements IClientContext { * Drain pending metrics and tear down owned resources. Called by * `TelemetryClientProvider.releaseClient` when refCount hits zero. */ - async close(): Promise { + async close(cancelPendingExports: boolean = false): Promise { if (this.closed) return; this.closed = true; this.logger.log(LogLevel.debug, `Closing TelemetryClient for host: ${this.host}`); @@ -316,7 +316,7 @@ class TelemetryClient implements IClientContext { this.logger.log(LogLevel.debug, `TelemetryClient aggregator close error: ${(err as Error).message}`); } try { - this.exporter.dispose(); + this.exporter.dispose(cancelPendingExports); } catch (err) { this.logger.log(LogLevel.debug, `TelemetryClient exporter dispose error: ${(err as Error).message}`); } diff --git a/lib/telemetry/TelemetryClientProvider.ts b/lib/telemetry/TelemetryClientProvider.ts index e042a63b..ff25a79a 100644 --- a/lib/telemetry/TelemetryClientProvider.ts +++ b/lib/telemetry/TelemetryClientProvider.ts @@ -28,6 +28,8 @@ interface TelemetryClientHolder { // in logs rather than silently growing the map. const MAX_CLIENTS_SOFT_LIMIT = 128; +const MAX_PENDING_CLOSES = 32; + /** * Process-wide registry of `TelemetryClient`s, one per host. Multiple * `DBSQLClient` instances connecting to the same host share the same @@ -47,6 +49,8 @@ class TelemetryClientProvider { private clients = new Map(); + private pendingCloses = new Set>(); + private softLimitWarned = false; // Production code should use `TelemetryClientProvider.getInstance()` for @@ -118,7 +122,7 @@ class TelemetryClientProvider { * Release `context`'s registration. When the last `DBSQLClient` releases, * the underlying `TelemetryClient.close()` runs and the entry is removed. */ - async releaseClient(context: IClientContext, host: string): Promise { + async releaseClient(context: IClientContext, host: string, waitForTelemetry: boolean = true): Promise { const logger = context.getLogger(); const key = TelemetryClientProvider.normalizeHostKey(host); const holder = this.clients.get(key); @@ -146,12 +150,24 @@ class TelemetryClientProvider { // getOrCreateClient creates a fresh instance rather than receiving // this closing one. this.clients.delete(key); - try { - await holder.client.close(); - logger.log(LogLevel.debug, `Closed and removed TelemetryClient for host: ${host}`); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - logger.log(LogLevel.debug, `Error releasing TelemetryClient: ${msg}`); + const background = !waitForTelemetry && this.pendingCloses.size < MAX_PENDING_CLOSES; + if (background) { + holder.client.getExporter().detachContext(); + } + const closing = (async () => { + try { + await holder.client.close(!waitForTelemetry); + logger.log(LogLevel.debug, `Closed and removed TelemetryClient for host: ${host}`); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + logger.log(LogLevel.debug, `Error releasing TelemetryClient: ${msg}`); + } + })(); + if (background) { + this.pendingCloses.add(closing); + closing.finally(() => this.pendingCloses.delete(closing)).catch(() => {}); + } else { + await closing; } } } diff --git a/tests/e2e/telemetry/close-e2e.test.ts b/tests/e2e/telemetry/close-e2e.test.ts new file mode 100644 index 00000000..106b3174 --- /dev/null +++ b/tests/e2e/telemetry/close-e2e.test.ts @@ -0,0 +1,55 @@ +import { expect } from 'chai'; +import { DBSQLClient } from '../../../lib'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; +import config from '../utils/config'; + +describe('client close telemetry lifecycle', function closeSuite() { + this.timeout(60000); + + for (const useKernel of [false, true]) { + for (const waitForTelemetry of [true, false]) { + it(`${ + useKernel ? 'kernel' : 'Thrift' + } closes and reopens with waitForTelemetry=${waitForTelemetry}`, async () => { + let errors = 0; + const client = new DBSQLClient({ + logger: { + log(level) { + if (level === LogLevel.error) errors += 1; + }, + }, + }); + try { + const cycle = async (): Promise => { + await client.connect({ host: config.host, path: config.path, token: config.token, ...{ useKernel } }); + const session = await client.openSession(); + const operation = await session.executeStatement( + 'SELECT id, CAST(NULL AS STRING) AS missing, id % 2 = 0 AS even FROM range(12) ORDER BY id LIMIT 3', + { maxRows: 2 }, + ); + const rows = (await operation.fetchAll({ maxRows: 2 })) as Array>; + expect(rows.map((row) => ({ ...row, id: Number(row.id) }))).to.deep.equal([ + { id: 0, missing: null, even: true }, + { id: 1, missing: null, even: false }, + { id: 2, missing: null, even: true }, + ]); + await operation.close(); + const early = await session.executeStatement('SELECT id FROM range(100) ORDER BY id', { maxRows: 1 }); + await early.fetchChunk({ maxRows: 1 }); + await early.close(); + const cancelled = await session.executeStatement('SELECT id FROM range(100)', { runAsync: true }); + await cancelled.cancel(); + await cancelled.close(); + await client.close({ waitForTelemetry }); + }; + await cycle(); + await cycle(); + await cycle(); + expect(errors).to.equal(0); + } finally { + await client.close(); + } + }); + } + } +}); diff --git a/tests/unit/telemetry/BackgroundClose.test.ts b/tests/unit/telemetry/BackgroundClose.test.ts new file mode 100644 index 00000000..1b0830b1 --- /dev/null +++ b/tests/unit/telemetry/BackgroundClose.test.ts @@ -0,0 +1,335 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import http from 'http'; +import { AddressInfo } from 'net'; +import DBSQLClient from '../../../lib/DBSQLClient'; +import TelemetryClientProvider from '../../../lib/telemetry/TelemetryClientProvider'; +import TelemetryClient from '../../../lib/telemetry/TelemetryClient'; +import { TelemetryEventType } from '../../../lib/telemetry/types'; +import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; +import ClientContextStub from '../.stubs/ClientContextStub'; + +describe('background telemetry close', () => { + const host = 'close-test.cloud.databricks.com'; + let clock: sinon.SinonFakeTimers; + let provider: TelemetryClientProvider; + + function addConnection(client: TelemetryClient, sessionId: string = 'session-1'): void { + client.getAggregator().processEvent({ + eventType: TelemetryEventType.CONNECTION_OPEN, + timestamp: Date.now(), + sessionId, + }); + } + + function delayedRequest(client: TelemetryClient, delay: number = 50): sinon.SinonStub { + return sinon.stub(client.getExporter() as any, 'sendRequest').callsFake( + () => + new Promise((resolve) => { + setTimeout(() => resolve({ ok: true, text: async () => '' }), delay); + }), + ); + } + + function makeContext(): ClientContextStub { + const context = new ClientContextStub({ telemetryCloseTimeoutMs: 100, telemetryMaxRetries: 0 }); + sinon.stub(context, 'getAuthProvider').returns({ authenticate: async () => ({ Authorization: 'Bearer mock' }) }); + return context; + } + + beforeEach(() => { + clock = sinon.useFakeTimers(); + TelemetryClientProvider.__resetInstanceForTests(); + provider = TelemetryClientProvider.getInstance(); + }); + + afterEach(() => { + sinon.restore(); + clock.restore(); + TelemetryClientProvider.__resetInstanceForTests(); + }); + + it('preserves the awaited default and explicit true', async () => { + for (const wait of [undefined, true]) { + const context = makeContext(); + const client = provider.getOrCreateClient(context, host); + const request = delayedRequest(client); + addConnection(client); + let released = false; + const closing = provider.releaseClient(context, host, wait).then(() => { + released = true; + }); + await clock.tickAsync(49); + expect(released).to.equal(false); + await clock.tickAsync(1); + await closing; + expect(request.calledOnce).to.equal(true); + } + expect(clock.countTimers()).to.equal(0); + }); + + it('detaches registry, auth and transport before DBSQLClient.close returns', async () => { + const context = makeContext(); + const client = new DBSQLClient({ logger: context.logger }); + Object.assign(client as any, { + host, + authProvider: context.getAuthProvider(), + connectionProvider: context.connectionProvider, + }); + const shared = provider.getOrCreateClient(client, host); + (client as any).telemetryClient = shared; + const request = delayedRequest(shared); + addConnection(shared); + + await client.close({ waitForTelemetry: false }); + + expect(provider.getActiveClients().size).to.equal(0); + expect(shared.isClosed()).to.equal(true); + expect((client as any).authProvider).to.equal(undefined); + expect((client as any).connectionProvider).to.equal(undefined); + expect(await (shared.getExporter() as any).connectionProvider).to.equal(context.connectionProvider); + expect((shared.getExporter() as any).authProvider).to.equal(context.getAuthProvider()); + await clock.tickAsync(50); + expect(request.calledOnce).to.equal(true); + await client.close({ waitForTelemetry: false }); + expect(clock.countTimers()).to.equal(0); + }); + + it('leaves other registrations active and close/reopen uses independent exporters', async () => { + const first = makeContext(); + const second = makeContext(); + const shared = provider.getOrCreateClient(first, host); + provider.getOrCreateClient(second, host); + const firstRequest = delayedRequest(shared); + addConnection(shared); + await provider.releaseClient(first, host, false); + expect(shared.isClosed()).to.equal(false); + expect(provider.getRefCount(host)).to.equal(1); + expect(firstRequest.called).to.equal(false); + + await provider.releaseClient(second, host, false); + const reopened = provider.getOrCreateClient(first, host); + const secondRequest = delayedRequest(reopened); + expect(reopened).not.to.equal(shared); + expect(reopened.isClosed()).to.equal(false); + addConnection(reopened, 'session-2'); + await provider.releaseClient(first, host, false); + await clock.tickAsync(50); + expect(firstRequest.calledOnce).to.equal(true); + expect(secondRequest.calledOnce).to.equal(true); + expect(provider.getActiveClients().size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + }); + + it('bounds detached exporters and waits when the background queue is full', async () => { + for (let index = 0; index < 32; index += 1) { + const context = makeContext(); + const client = provider.getOrCreateClient(context, host); + delayedRequest(client); + addConnection(client); + await provider.releaseClient(context, host, false); + } + expect((provider as any).pendingCloses.size).to.equal(32); + const context = makeContext(); + const client = provider.getOrCreateClient(context, host); + delayedRequest(client); + addConnection(client); + let released = false; + const closing = provider.releaseClient(context, host, false).then(() => { + released = true; + }); + await clock.tickAsync(49); + expect(released).to.equal(false); + expect((provider as any).pendingCloses.size).to.equal(32); + await clock.tickAsync(1); + await closing; + expect((provider as any).pendingCloses.size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + }); + + it('drains an in-flight batch and the metrics queued during that batch', async () => { + const context = makeContext(); + context.configOverrides.telemetryBatchSize = 1; + const client = provider.getOrCreateClient(context, host); + const request = delayedRequest(client, 20); + addConnection(client, 'first'); + await clock.tickAsync(0); + addConnection(client, 'second'); + await provider.releaseClient(context, host, false); + await clock.tickAsync(40); + expect(request.callCount).to.equal(2); + const sessions = request.getCalls().map((call) => { + const body = JSON.parse(call.args[1].body); + return JSON.parse(body.protoLogs[0]).entry.sql_driver_log.session_id; + }); + expect(sessions).to.deep.equal(['first', 'second']); + expect(clock.countTimers()).to.equal(0); + }); + + it('aborts a stalled export at the deadline without error logs or leaked timers', async () => { + const context = makeContext(); + const log = sinon.spy(context.logger, 'log'); + const client = provider.getOrCreateClient(context, host); + let signal: AbortSignal | undefined; + const request = sinon.stub(client.getExporter() as any, 'sendRequest').callsFake( + (_url: unknown, init: any) => + new Promise((_resolve, reject) => { + signal = init.signal; + signal!.addEventListener('abort', () => reject(new Error('cancelled')), { once: true }); + }), + ); + addConnection(client); + await provider.releaseClient(context, host, false); + await clock.tickAsync(99); + expect(signal?.aborted).to.equal(false); + await clock.tickAsync(1); + expect(signal?.aborted).to.equal(true); + await clock.tickAsync(60000); + expect(request.calledOnce).to.equal(true); + expect(log.getCalls().filter((call) => call.args[0] === LogLevel.error)).to.have.length(0); + expect((provider as any).pendingCloses.size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + }); + + it('cancels retry backoff when the background deadline expires', async () => { + const context = makeContext(); + Object.assign(context.configOverrides, { telemetryMaxRetries: 3, telemetryBackoffBaseMs: 1000 }); + const client = provider.getOrCreateClient(context, host); + const request = sinon.stub(client.getExporter() as any, 'sendRequest').resolves({ + ok: false, + status: 503, + statusText: 'Unavailable', + text: async () => '', + }); + addConnection(client); + await provider.releaseClient(context, host, false); + await clock.tickAsync(100); + await clock.tickAsync(60000); + expect(request.calledOnce).to.equal(true); + expect(clock.countTimers()).to.equal(0); + expect((provider as any).pendingCloses.size).to.equal(0); + }); + + it('handles a rejected detached transport without an unhandled rejection', async () => { + const context = makeContext(); + sinon.stub(context, 'getConnectionProvider').rejects(new Error('transport unavailable')); + const client = provider.getOrCreateClient(context, host); + addConnection(client); + await provider.releaseClient(context, host, false); + await clock.tickAsync(100); + expect((provider as any).pendingCloses.size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + }); +}); + +describe('background close HTTP transport', () => { + it('uses the detached transport and credentials after the owning client is cleared', async () => { + let finishAuth!: () => void; + const authenticated = new Promise((resolve) => { + finishAuth = resolve; + }); + let received!: (value: { authorization?: string; body: string }) => void; + const exported = new Promise<{ authorization?: string; body: string }>((resolve) => { + received = resolve; + }); + const server = http.createServer((request, response) => { + let body = ''; + request.on('data', (chunk) => { + body += chunk; + }); + request.on('end', () => { + response.end('{}'); + received({ authorization: request.headers.authorization, body }); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const provider = new TelemetryClientProvider(); + const context = new ClientContextStub(); + const client = new DBSQLClient({ logger: context.logger }); + const agent = sinon.spy(context.connectionProvider, 'getAgent'); + Object.assign(client as any, { + host: 'detached.cloud.databricks.com', + authProvider: { + authenticate: async () => { + await authenticated; + return { Authorization: 'Bearer before-close' }; + }, + }, + connectionProvider: context.connectionProvider, + }); + const shared = provider.getOrCreateClient(client, 'detached.cloud.databricks.com'); + const exporter = shared.getExporter() as any; + const sendRequest = exporter.sendRequest.bind(exporter); + const stub = sinon + .stub(exporter, 'sendRequest') + .callsFake((_url: unknown, init: unknown) => + sendRequest(`http://127.0.0.1:${(server.address() as AddressInfo).port}/telemetry-ext`, init), + ); + shared.getAggregator().processEvent({ + eventType: TelemetryEventType.CONNECTION_OPEN, + sessionId: 'detached-session', + timestamp: Date.now(), + }); + try { + await provider.releaseClient(client, 'detached.cloud.databricks.com', false); + Object.assign(client as any, { authProvider: undefined, connectionProvider: undefined }); + finishAuth(); + const request = await exported; + expect(request.authorization).to.equal('Bearer before-close'); + const body = JSON.parse(request.body); + expect(JSON.parse(body.protoLogs[0]).entry.sql_driver_log.session_id).to.equal('detached-session'); + expect(agent.calledOnce).to.equal(true); + const pending = [...(provider as any).pendingCloses] as Promise[]; + await Promise.all(pending); + } finally { + stub.restore(); + agent.restore(); + finishAuth(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it('does not start a request when authentication finishes after the close deadline', async () => { + let requests = 0; + const server = http.createServer((_request, response) => { + requests += 1; + response.end('{}'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + let finishAuth!: () => void; + const authenticated = new Promise((resolve) => { + finishAuth = resolve; + }); + const context = new ClientContextStub({ telemetryCloseTimeoutMs: 10 }); + const auth = sinon.stub(context, 'getAuthProvider').returns({ + authenticate: async () => { + await authenticated; + return { Authorization: 'Bearer late' }; + }, + }); + const provider = new TelemetryClientProvider(); + const shared = provider.getOrCreateClient(context, 'late.cloud.databricks.com'); + const exporter = shared.getExporter() as any; + const sendRequest = exporter.sendRequest.bind(exporter); + const stub = sinon + .stub(exporter, 'sendRequest') + .callsFake((_url: unknown, init: unknown) => + sendRequest(`http://127.0.0.1:${(server.address() as AddressInfo).port}/telemetry-ext`, init), + ); + shared.getAggregator().processEvent({ eventType: TelemetryEventType.CONNECTION_OPEN, timestamp: Date.now() }); + try { + await provider.releaseClient(context, 'late.cloud.databricks.com', false); + await Promise.all([...(provider as any).pendingCloses]); + finishAuth(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(requests).to.equal(0); + } finally { + finishAuth(); + stub.restore(); + auth.restore(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); From 42e0c99a7edb5552ae8d24e55645a6afb2492628 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 08:40:07 +0000 Subject: [PATCH 2/3] Keep detached telemetry authentication bound to its connection Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/DBSQLClient.ts | 15 +- .../auth/DatabricksOAuth/OAuthManager.ts | 4 +- lib/contracts/IDBSQLClient.ts | 8 +- tests/e2e/telemetry/close-e2e.test.ts | 1 + tests/unit/telemetry/BackgroundClose.test.ts | 188 +++++++++++++++++- 6 files changed, 204 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11711a3c..cb590b56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Add opt-in `client.close({ waitForTelemetry: false })` for bounded background Thrift telemetry export. The default still awaits export; explicit process exit or runtime suspension can lose the background batch. +Add opt-in `client.close({ waitForTelemetry: false })` for background Thrift telemetry export; the default remains awaited. Background mode abandons exports and retries still active at `telemetryCloseTimeoutMs` (2s by default), even in a live process; explicit exit or runtime suspension can also lose batches, and no public hook drains detached exports. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index aaa000bd..29aa9881 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -329,6 +329,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I useDatabricksOAuthInAzure: options.useDatabricksOAuthInAzure, scopes: options.oauthScopes, context: this, + connectionProvider: this.connectionProvider, }); case 'custom': return options.provider; @@ -381,10 +382,11 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I // Optionally wrap with federation if (enableFederation) { + const { connectionProvider } = this; wrapped = new FederationProvider(wrapped, host, { clientId: federationClientId, // The connection's agent carries `proxy` and the TLS options. - getAgent: async (url) => (await this.getConnectionProvider()).getAgent(url), + getAgent: async (url) => (connectionProvider ?? (await this.getConnectionProvider())).getAgent(url), }); } @@ -782,10 +784,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I // hit endpoints that don't carry the workspace in their URL path. this.config.customHeaders = this.buildCustomHeaders(options.path, options.customHeaders); - this.authProvider = this.createAuthProvider(options, authProvider); - this.connectionProvider = this.createConnectionProvider(options); + this.authProvider = this.createAuthProvider(options, authProvider); + // M0: `useKernel` is consumed via a non-exported internal-options cast so it // doesn't ship in the public `.d.ts`. Mirrors Python's `kwargs.get("use_kernel")` // pattern (see databricks-sql-python/src/databricks/sql/session.py). @@ -952,9 +954,12 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I * without calling `close()`. Long-lived applications should `await` this * method on shutdown so the aggregator drains its remaining metrics. * `waitForTelemetry: false` opts into a bounded background Thrift export; - * explicit process exit or runtime suspension can lose the final batch. + * exports and retries still active at the close deadline are abandoned even + * in a live process. Explicit exit or runtime suspension can also lose batches. + * No public hook drains previously detached batches. */ public async close(options: ClientCloseOptions = {}): Promise { + const waitForTelemetry = options?.waitForTelemetry !== false; try { await this.sessions.closeAll(); await this.backend?.close(); @@ -966,7 +971,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I if (this.host && this.telemetryClient) { try { - await TelemetryClientProvider.getInstance().releaseClient(this, this.host, options.waitForTelemetry); + await TelemetryClientProvider.getInstance().releaseClient(this, this.host, waitForTelemetry); } catch (error: any) { this.logger.log(LogLevel.debug, `Telemetry cleanup error: ${error.message}`); } diff --git a/lib/connection/auth/DatabricksOAuth/OAuthManager.ts b/lib/connection/auth/DatabricksOAuth/OAuthManager.ts index 40ff80ae..8fa13713 100644 --- a/lib/connection/auth/DatabricksOAuth/OAuthManager.ts +++ b/lib/connection/auth/DatabricksOAuth/OAuthManager.ts @@ -6,6 +6,7 @@ import OAuthToken from './OAuthToken'; import AuthorizationCode from './AuthorizationCode'; import { OAuthScope, OAuthScopes, defaultOAuthScopes, scopeDelimiter } from './OAuthScope'; import IClientContext from '../../../contracts/IClientContext'; +import IConnectionProvider from '../../contracts/IConnectionProvider'; export enum OAuthFlow { U2M = 'U2M', @@ -21,6 +22,7 @@ export interface OAuthManagerOptions { clientSecret?: string; useDatabricksOAuthInAzure?: boolean; context: IClientContext; + connectionProvider?: IConnectionProvider; } function getDatabricksOIDCUrl(host: string): string { @@ -61,7 +63,7 @@ export default abstract class OAuthManager { protected async getClient(): Promise { // Obtain http agent each time when we need an OAuth client // to ensure that we always use a valid agent instance - const connectionProvider = await this.context.getConnectionProvider(); + const connectionProvider = this.options.connectionProvider ?? (await this.context.getConnectionProvider()); // openid-client picks `http`/`https` by each request's URL, so the agent must // match that URL's protocol rather than the SQL transport's. this.agent = await connectionProvider.getAgent('https:'); diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 3519e499..5b01caec 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -14,9 +14,11 @@ export interface ClientCloseOptions { /** * Wait for the final Thrift telemetry export (default true). With false, * export runs in the background for up to telemetryCloseTimeoutMs (2s by - * default); a full background queue falls back to waiting. process.exit() - * or a frozen runtime can lose that batch. Native kernel session cleanup, - * including its telemetry flush, is always awaited. + * default); a full background queue falls back to waiting. Exports and retries + * still active at the deadline are abandoned even in a live process. Explicit + * process.exit() or runtime suspension can also lose batches. No public hook + * drains previously detached batches. Native kernel session cleanup, including + * its telemetry flush, is always awaited. */ waitForTelemetry?: boolean; } diff --git a/tests/e2e/telemetry/close-e2e.test.ts b/tests/e2e/telemetry/close-e2e.test.ts index 106b3174..2271ba4e 100644 --- a/tests/e2e/telemetry/close-e2e.test.ts +++ b/tests/e2e/telemetry/close-e2e.test.ts @@ -22,6 +22,7 @@ describe('client close telemetry lifecycle', function closeSuite() { try { const cycle = async (): Promise => { await client.connect({ host: config.host, path: config.path, token: config.token, ...{ useKernel } }); + if (!useKernel) expect(client.getTelemetryStats()).not.to.equal(undefined); const session = await client.openSession(); const operation = await session.executeStatement( 'SELECT id, CAST(NULL AS STRING) AS missing, id % 2 = 0 AS even FROM range(12) ORDER BY id LIMIT 3', diff --git a/tests/unit/telemetry/BackgroundClose.test.ts b/tests/unit/telemetry/BackgroundClose.test.ts index 1b0830b1..3658f8c8 100644 --- a/tests/unit/telemetry/BackgroundClose.test.ts +++ b/tests/unit/telemetry/BackgroundClose.test.ts @@ -2,12 +2,20 @@ import { expect } from 'chai'; import sinon from 'sinon'; import http from 'http'; import { AddressInfo } from 'net'; +import fetch, { RequestInfo, RequestInit } from 'node-fetch'; +import { Issuer } from 'openid-client'; import DBSQLClient from '../../../lib/DBSQLClient'; +import FeatureFlagCache from '../../../lib/FeatureFlagCache'; +import FederationProvider, { + setFederationFetchForTest, +} from '../../../lib/connection/auth/tokenProvider/FederationProvider'; +import OAuthToken from '../../../lib/connection/auth/DatabricksOAuth/OAuthToken'; import TelemetryClientProvider from '../../../lib/telemetry/TelemetryClientProvider'; import TelemetryClient from '../../../lib/telemetry/TelemetryClient'; import { TelemetryEventType } from '../../../lib/telemetry/types'; import { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; import ClientContextStub from '../.stubs/ClientContextStub'; +import { createAccessToken } from '../.stubs/OAuth'; describe('background telemetry close', () => { const host = 'close-test.cloud.databricks.com'; @@ -95,6 +103,35 @@ describe('background telemetry close', () => { expect(clock.countTimers()).to.equal(0); }); + for (const options of [null, 0, { waitForTelemetry: null }, { waitForTelemetry: 0 }, { waitForTelemetry: '' }]) { + it(`requires explicit false to detach telemetry: ${JSON.stringify(options)}`, async () => { + const context = makeContext(); + const client = new DBSQLClient({ logger: context.logger }); + Object.assign(client, { + host, + authProvider: context.getAuthProvider(), + connectionProvider: context.connectionProvider, + }); + const shared = provider.getOrCreateClient(client, host); + Object.assign(client, { telemetryClient: shared }); + const request = delayedRequest(shared); + addConnection(shared); + let closed = false; + const closing = client.close(options as any).then(() => { + closed = true; + }); + await clock.tickAsync(49); + expect(closed).to.equal(false); + expect(provider.getRefCount(host)).to.equal(0); + await clock.tickAsync(1); + await closing; + expect(request.calledOnce).to.equal(true); + expect(shared.isClosed()).to.equal(true); + expect(provider.getActiveClients().size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + }); + } + it('leaves other registrations active and close/reopen uses independent exporters', async () => { const first = makeContext(); const second = makeContext(); @@ -184,6 +221,8 @@ describe('background telemetry close', () => { expect(signal?.aborted).to.equal(false); await clock.tickAsync(1); expect(signal?.aborted).to.equal(true); + expect(clock.countTimers()).to.equal(0); + expect(log.getCalls().some((call) => String(call.args[1]).startsWith('Telemetry export error:'))).to.equal(false); await clock.tickAsync(60000); expect(request.calledOnce).to.equal(true); expect(log.getCalls().filter((call) => call.args[0] === LogLevel.error)).to.have.length(0); @@ -204,6 +243,7 @@ describe('background telemetry close', () => { addConnection(client); await provider.releaseClient(context, host, false); await clock.tickAsync(100); + expect(clock.countTimers()).to.equal(0); await clock.tickAsync(60000); expect(request.calledOnce).to.equal(true); expect(clock.countTimers()).to.equal(0); @@ -212,14 +252,154 @@ describe('background telemetry close', () => { it('handles a rejected detached transport without an unhandled rejection', async () => { const context = makeContext(); + const unhandled: unknown[] = []; + const onRejection = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on('unhandledRejection', onRejection); + sinon + .stub(context.getAuthProvider()!, 'authenticate') + .callsFake(() => new Promise((resolve) => setTimeout(() => resolve({ Authorization: 'Bearer mock' }), 50))); sinon.stub(context, 'getConnectionProvider').rejects(new Error('transport unavailable')); const client = provider.getOrCreateClient(context, host); addConnection(client); - await provider.releaseClient(context, host, false); - await clock.tickAsync(100); - expect((provider as any).pendingCloses.size).to.equal(0); - expect(clock.countTimers()).to.equal(0); + try { + await provider.releaseClient(context, host, false); + await clock.tickAsync(25); + expect(unhandled).to.deep.equal([]); + await clock.tickAsync(75); + expect(unhandled).to.deep.equal([]); + expect((provider as any).pendingCloses.size).to.equal(0); + expect(clock.countTimers()).to.equal(0); + } finally { + process.off('unhandledRejection', onRejection); + } + }); +}); + +describe('background close built-in authentication', () => { + const host = 'close-auth.cloud.databricks.com'; + let server: http.Server; + let endpoint: string; + let client: DBSQLClient; + let requests: Array<{ path?: string; authorization?: string; body: string }>; + let finishAuth: () => void; + let authenticated: Promise; + const refreshedToken = createAccessToken(Math.floor(Date.now() / 1000) + 3600); + + beforeEach(async () => { + TelemetryClientProvider.__resetInstanceForTests(); + sinon.stub(FeatureFlagCache.prototype as any, 'fetchFeatureFlags').resolves(new Map()); + sinon.stub(FeatureFlagCache.prototype, 'getBoolean').resolves(true); + authenticated = new Promise((resolve) => { + finishAuth = resolve; + }); + requests = []; + server = http.createServer((request, response) => { + let body = ''; + request.on('data', (chunk) => { + body += chunk; + }); + request.on('end', () => { + requests.push({ path: request.url, authorization: request.headers.authorization, body }); + response.setHeader('Content-Type', 'application/json'); + response.end( + request.url === '/oidc/v1/token' + ? JSON.stringify({ access_token: refreshedToken, refresh_token: 'next-refresh', token_type: 'Bearer' }) + : '{}', + ); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + client = new DBSQLClient({ logger: new ClientContextStub().logger }); + }); + + afterEach(async () => { + finishAuth(); + await client.close(); + await Promise.all([...(TelemetryClientProvider.getInstance() as any).pendingCloses]); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + setFederationFetchForTest(); + sinon.restore(); + TelemetryClientProvider.__resetInstanceForTests(); }); + + async function closeAndExport(): Promise { + const shared = (client as any).telemetryClient as TelemetryClient; + expect(shared).not.to.equal(undefined); + const exporter = shared.getExporter() as any; + const sendRequest = exporter.sendRequest.bind(exporter); + sinon + .stub(exporter, 'sendRequest') + .callsFake((_url: unknown, init: unknown) => sendRequest(`${endpoint}/telemetry-ext`, init)); + shared.getAggregator().processEvent({ + eventType: TelemetryEventType.CONNECTION_OPEN, + timestamp: Date.now(), + sessionId: 'built-in-auth', + }); + await client.close({ waitForTelemetry: false }); + expect((client as any).connectionProvider).to.equal(undefined); + finishAuth(); + await Promise.all([...(TelemetryClientProvider.getInstance() as any).pendingCloses]); + expect(requests.map((request) => request.path)).to.deep.equal(['/oidc/v1/token', '/telemetry-ext']); + expect(requests[1].authorization).to.equal(`Bearer ${refreshedToken}`); + } + + it('exchanges a foreign JWT after close instead of exporting the original token', async () => { + const getToken = FederationProvider.prototype.getToken; + sinon + .stub(FederationProvider.prototype, 'getToken') + .callsFake(async function delayedFederation(this: FederationProvider) { + await authenticated; + return getToken.call(this); + }); + const externalToken = `header.${Buffer.from(JSON.stringify({ iss: 'https://external-idp.example' })).toString( + 'base64url', + )}.signature`; + await client.connect({ + host, + path: '/', + authType: 'static-token', + staticToken: externalToken, + enableTokenFederation: true, + }); + const transport = await client.getConnectionProvider(); + const federationFetch = async (url: RequestInfo, init?: RequestInit) => { + expect(init?.agent).to.equal(await transport.getAgent(String(url))); + return fetch(`${endpoint}/oidc/v1/token`, { ...init, agent: await transport.getAgent(endpoint) }); + }; + setFederationFetchForTest(federationFetch as unknown as typeof fetch); + await closeAndExport(); + expect(new URLSearchParams(requests[0].body).get('subject_token')).to.equal(externalToken); + }); + + for (const oauthClientSecret of [undefined, 'mock-secret']) { + it(`refreshes expired OAuth ${oauthClientSecret ? 'M2M' : 'U2M'} after close`, async () => { + sinon + .stub(Issuer, 'discover') + .resolves(new Issuer({ issuer: `https://${host}/oidc`, token_endpoint: `${endpoint}/oidc/v1/token` })); + await client.connect({ + host, + path: '/', + authType: 'databricks-oauth', + oauthClientId: 'mock-client', + oauthClientSecret, + persistence: { + read: async () => { + await authenticated; + return new OAuthToken(createAccessToken(1), 'previous-refresh'); + }, + persist: async () => {}, + }, + }); + await closeAndExport(); + expect(new URLSearchParams(requests[0].body).get('grant_type')).to.equal( + oauthClientSecret ? 'client_credentials' : 'refresh_token', + ); + }); + } }); describe('background close HTTP transport', () => { From e4502c429fa5be02946e1c6da5f7d10b550f8f50 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 09:38:40 +0000 Subject: [PATCH 3/3] Isolate detached telemetry state and retain awaited authentication Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/DBSQLClient.ts | 16 +- .../auth/DatabricksOAuth/OAuthManager.ts | 4 +- lib/contracts/IDBSQLClient.ts | 6 +- lib/telemetry/DatabricksTelemetryExporter.ts | 33 ++- lib/telemetry/TelemetryClient.ts | 12 +- lib/telemetry/TelemetryClientProvider.ts | 8 +- tests/unit/telemetry/BackgroundClose.test.ts | 228 ++++++++++++++---- 8 files changed, 234 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb590b56..1743fe47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Add opt-in `client.close({ waitForTelemetry: false })` for background Thrift telemetry export; the default remains awaited. Background mode abandons exports and retries still active at `telemetryCloseTimeoutMs` (2s by default), even in a live process; explicit exit or runtime suspension can also lose batches, and no public hook drains detached exports. +Add opt-in `client.close({ waitForTelemetry: false })` for background Thrift telemetry export with built-in access-token/PAT auth; OAuth, token-provider/federation and custom auth retain the unchanged awaited path. Background mode abandons exports and retries at `telemetryCloseTimeoutMs` (2s by default) even in a live process, exit or suspension can also lose batches, and no public hook drains detached exports. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index 29aa9881..c491984e 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -329,7 +329,6 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I useDatabricksOAuthInAzure: options.useDatabricksOAuthInAzure, scopes: options.oauthScopes, context: this, - connectionProvider: this.connectionProvider, }); case 'custom': return options.provider; @@ -382,11 +381,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I // Optionally wrap with federation if (enableFederation) { - const { connectionProvider } = this; wrapped = new FederationProvider(wrapped, host, { clientId: federationClientId, // The connection's agent carries `proxy` and the TLS options. - getAgent: async (url) => (connectionProvider ?? (await this.getConnectionProvider())).getAgent(url), + getAgent: async (url) => (await this.getConnectionProvider()).getAgent(url), }); } @@ -784,10 +782,10 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I // hit endpoints that don't carry the workspace in their URL path. this.config.customHeaders = this.buildCustomHeaders(options.path, options.customHeaders); - this.connectionProvider = this.createConnectionProvider(options); - this.authProvider = this.createAuthProvider(options, authProvider); + this.connectionProvider = this.createConnectionProvider(options); + // M0: `useKernel` is consumed via a non-exported internal-options cast so it // doesn't ship in the public `.d.ts`. Mirrors Python's `kwargs.get("use_kernel")` // pattern (see databricks-sql-python/src/databricks/sql/session.py). @@ -953,9 +951,11 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I * telemetry buffered between flush ticks is lost if the process exits * without calling `close()`. Long-lived applications should `await` this * method on shutdown so the aggregator drains its remaining metrics. - * `waitForTelemetry: false` opts into a bounded background Thrift export; - * exports and retries still active at the close deadline are abandoned even - * in a live process. Explicit exit or runtime suspension can also lose batches. + * `waitForTelemetry: false` enables background Thrift export only for built-in + * access-token/PAT auth. OAuth, token-provider/federation and custom auth retain + * the awaited path, without detaching exchange or refresh work. + * Background exports and retries still active at the close deadline are abandoned + * even in a live process. Explicit exit or runtime suspension can also lose batches. * No public hook drains previously detached batches. */ public async close(options: ClientCloseOptions = {}): Promise { diff --git a/lib/connection/auth/DatabricksOAuth/OAuthManager.ts b/lib/connection/auth/DatabricksOAuth/OAuthManager.ts index 8fa13713..40ff80ae 100644 --- a/lib/connection/auth/DatabricksOAuth/OAuthManager.ts +++ b/lib/connection/auth/DatabricksOAuth/OAuthManager.ts @@ -6,7 +6,6 @@ import OAuthToken from './OAuthToken'; import AuthorizationCode from './AuthorizationCode'; import { OAuthScope, OAuthScopes, defaultOAuthScopes, scopeDelimiter } from './OAuthScope'; import IClientContext from '../../../contracts/IClientContext'; -import IConnectionProvider from '../../contracts/IConnectionProvider'; export enum OAuthFlow { U2M = 'U2M', @@ -22,7 +21,6 @@ export interface OAuthManagerOptions { clientSecret?: string; useDatabricksOAuthInAzure?: boolean; context: IClientContext; - connectionProvider?: IConnectionProvider; } function getDatabricksOIDCUrl(host: string): string { @@ -63,7 +61,7 @@ export default abstract class OAuthManager { protected async getClient(): Promise { // Obtain http agent each time when we need an OAuth client // to ensure that we always use a valid agent instance - const connectionProvider = this.options.connectionProvider ?? (await this.context.getConnectionProvider()); + const connectionProvider = await this.context.getConnectionProvider(); // openid-client picks `http`/`https` by each request's URL, so the agent must // match that URL's protocol rather than the SQL transport's. this.agent = await connectionProvider.getAgent('https:'); diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 5b01caec..fd401827 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -13,8 +13,10 @@ export interface ClientOptions { export interface ClientCloseOptions { /** * Wait for the final Thrift telemetry export (default true). With false, - * export runs in the background for up to telemetryCloseTimeoutMs (2s by - * default); a full background queue falls back to waiting. Exports and retries + * built-in access-token/PAT auth exports in the background up to + * telemetryCloseTimeoutMs (2s by default). All other auth providers (OAuth, + * token adapters/federation, custom) and a full queue retain the awaited path; no + * exchange or refresh work is detached. Background exports and retries * still active at the deadline are abandoned even in a live process. Explicit * process.exit() or runtime suspension can also lose batches. No public hook * drains previously detached batches. Native kernel session cleanup, including diff --git a/lib/telemetry/DatabricksTelemetryExporter.ts b/lib/telemetry/DatabricksTelemetryExporter.ts index b33d6d64..d5a3c4d9 100644 --- a/lib/telemetry/DatabricksTelemetryExporter.ts +++ b/lib/telemetry/DatabricksTelemetryExporter.ts @@ -20,6 +20,7 @@ import IClientContext from '../contracts/IClientContext'; import { LogLevel } from '../contracts/IDBSQLLogger'; import IAuthentication from '../connection/contracts/IAuthentication'; import IConnectionProvider from '../connection/contracts/IConnectionProvider'; +import PlainHttpAuthentication from '../connection/auth/PlainHttpAuthentication'; import AuthenticationError from '../errors/AuthenticationError'; import HiveDriverError from '../errors/HiveDriverError'; import { TelemetryMetric, DEFAULT_TELEMETRY_CONFIG } from './types'; @@ -126,7 +127,7 @@ export default class DatabricksTelemetryExporter { private authMissingWarned = false; - private readonly abortController = new AbortController(); + private abortController?: AbortController; private connectionProvider?: Promise; @@ -161,17 +162,23 @@ export default class DatabricksTelemetryExporter { * the last consumer on its host; multi-consumer refcounting on the * registry will land in the consumer-wiring PR. */ - dispose(cancelPendingExports: boolean = false): void { - if (cancelPendingExports) { - this.abortController.abort(); - } + dispose(): void { + this.abortController?.abort(); this.circuitBreakerRegistry.removeCircuitBreaker(this.host); } - detachContext(): void { - this.authProvider = this.authProvider ?? this.context.getAuthProvider?.(); + detachContext(): boolean { + const authProvider = this.authProvider ?? this.context.getAuthProvider?.(); + if ( + !(authProvider instanceof PlainHttpAuthentication) || + authProvider.authenticate !== PlainHttpAuthentication.prototype.authenticate + ) + return false; + this.authProvider = authProvider; + this.abortController = new AbortController(); this.connectionProvider = this.context.getConnectionProvider(); this.connectionProvider.catch(() => {}); + return true; } async export(metrics: TelemetryMetric[]): Promise { @@ -221,12 +228,12 @@ export default class DatabricksTelemetryExporter { /* eslint-disable no-await-in-loop */ for (let attempt = 0; attempt < totalAttempts; attempt += 1) { - if (this.abortController.signal.aborted) return; + if (this.abortController?.signal.aborted) return; try { await this.exportInternal(metrics); return; } catch (error: any) { - if (this.abortController.signal.aborted) return; + if (this.abortController?.signal.aborted) return; lastError = error; if ( @@ -315,7 +322,7 @@ export default class DatabricksTelemetryExporter { headers, body, timeout: 10000, - signal: this.abortController.signal as unknown as RequestInit['signal'], + ...(this.abortController ? { signal: this.abortController.signal as unknown as RequestInit['signal'] } : {}), }); if (!response.ok) { @@ -488,8 +495,12 @@ export default class DatabricksTelemetryExporter { } private sleep(ms: number): Promise { + if (!this.abortController) + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + const { signal } = this.abortController; return new Promise((resolve) => { - const { signal } = this.abortController; let timer: NodeJS.Timeout; const finish = (): void => { clearTimeout(timer); diff --git a/lib/telemetry/TelemetryClient.ts b/lib/telemetry/TelemetryClient.ts index a2dc3fc8..8ab1f745 100644 --- a/lib/telemetry/TelemetryClient.ts +++ b/lib/telemetry/TelemetryClient.ts @@ -49,7 +49,7 @@ class TelemetryClient implements IClientContext { private readonly logger: IDBSQLLogger; - private readonly config: ClientConfig; + private config: ClientConfig; private readonly circuitBreakerRegistry: CircuitBreakerRegistry; @@ -302,11 +302,17 @@ class TelemetryClient implements IClientContext { return this.closed; } + detachContext(): boolean { + if (!this.exporter.detachContext()) return false; + this.config = { ...this.config, customHeaders: this.config.customHeaders && { ...this.config.customHeaders } }; + return true; + } + /** * Drain pending metrics and tear down owned resources. Called by * `TelemetryClientProvider.releaseClient` when refCount hits zero. */ - async close(cancelPendingExports: boolean = false): Promise { + async close(): Promise { if (this.closed) return; this.closed = true; this.logger.log(LogLevel.debug, `Closing TelemetryClient for host: ${this.host}`); @@ -316,7 +322,7 @@ class TelemetryClient implements IClientContext { this.logger.log(LogLevel.debug, `TelemetryClient aggregator close error: ${(err as Error).message}`); } try { - this.exporter.dispose(cancelPendingExports); + this.exporter.dispose(); } catch (err) { this.logger.log(LogLevel.debug, `TelemetryClient exporter dispose error: ${(err as Error).message}`); } diff --git a/lib/telemetry/TelemetryClientProvider.ts b/lib/telemetry/TelemetryClientProvider.ts index ff25a79a..1c2189a0 100644 --- a/lib/telemetry/TelemetryClientProvider.ts +++ b/lib/telemetry/TelemetryClientProvider.ts @@ -150,13 +150,11 @@ class TelemetryClientProvider { // getOrCreateClient creates a fresh instance rather than receiving // this closing one. this.clients.delete(key); - const background = !waitForTelemetry && this.pendingCloses.size < MAX_PENDING_CLOSES; - if (background) { - holder.client.getExporter().detachContext(); - } + const background = + waitForTelemetry === false && this.pendingCloses.size < MAX_PENDING_CLOSES && holder.client.detachContext(); const closing = (async () => { try { - await holder.client.close(!waitForTelemetry); + await holder.client.close(); logger.log(LogLevel.debug, `Closed and removed TelemetryClient for host: ${host}`); } catch (error) { const msg = error instanceof Error ? error.message : String(error); diff --git a/tests/unit/telemetry/BackgroundClose.test.ts b/tests/unit/telemetry/BackgroundClose.test.ts index 3658f8c8..ee8fd75b 100644 --- a/tests/unit/telemetry/BackgroundClose.test.ts +++ b/tests/unit/telemetry/BackgroundClose.test.ts @@ -10,6 +10,7 @@ import FederationProvider, { setFederationFetchForTest, } from '../../../lib/connection/auth/tokenProvider/FederationProvider'; import OAuthToken from '../../../lib/connection/auth/DatabricksOAuth/OAuthToken'; +import PlainHttpAuthentication from '../../../lib/connection/auth/PlainHttpAuthentication'; import TelemetryClientProvider from '../../../lib/telemetry/TelemetryClientProvider'; import TelemetryClient from '../../../lib/telemetry/TelemetryClient'; import { TelemetryEventType } from '../../../lib/telemetry/types'; @@ -41,7 +42,7 @@ describe('background telemetry close', () => { function makeContext(): ClientContextStub { const context = new ClientContextStub({ telemetryCloseTimeoutMs: 100, telemetryMaxRetries: 0 }); - sinon.stub(context, 'getAuthProvider').returns({ authenticate: async () => ({ Authorization: 'Bearer mock' }) }); + sinon.stub(context, 'getAuthProvider').returns(new PlainHttpAuthentication({ context, password: 'mock' })); return context; } @@ -60,6 +61,8 @@ describe('background telemetry close', () => { it('preserves the awaited default and explicit true', async () => { for (const wait of [undefined, true]) { const context = makeContext(); + const config = context.getConfig(); + sinon.stub(context, 'getConfig').returns(config); const client = provider.getOrCreateClient(context, host); const request = delayedRequest(client); addConnection(client); @@ -72,6 +75,9 @@ describe('background telemetry close', () => { await clock.tickAsync(1); await closing; expect(request.calledOnce).to.equal(true); + expect(client.getConfig()).to.equal(config); + expect((client.getExporter() as any).abortController).to.equal(undefined); + expect(request.firstCall.args[1]).not.to.have.property('signal'); } expect(clock.countTimers()).to.equal(0); }); @@ -103,6 +109,33 @@ describe('background telemetry close', () => { expect(clock.countTimers()).to.equal(0); }); + for (const overriddenBuiltin of [false, true]) { + it(`awaits ${overriddenBuiltin ? 'overridden built-in' : 'custom'} auth without a detached close`, async () => { + const context = makeContext(); + const authenticate = async () => ({ Authorization: 'Bearer custom' }); + const auth = overriddenBuiltin + ? Object.assign(new PlainHttpAuthentication({ context, password: 'mock' }), { authenticate }) + : { authenticate }; + (context.getAuthProvider as sinon.SinonStub).returns(auth); + const client = provider.getOrCreateClient(context, host); + const request = delayedRequest(client); + addConnection(client); + let closed = false; + const closing = provider.releaseClient(context, host, false).then(() => { + closed = true; + }); + await clock.tickAsync(49); + expect(closed).to.equal(false); + expect((provider as any).pendingCloses.size).to.equal(0); + expect((client.getExporter() as any).abortController).to.equal(undefined); + await clock.tickAsync(1); + await closing; + expect(request.calledOnce).to.equal(true); + expect(request.firstCall.args[1]).not.to.have.property('signal'); + expect(clock.countTimers()).to.equal(0); + }); + } + for (const options of [null, 0, { waitForTelemetry: null }, { waitForTelemetry: 0 }, { waitForTelemetry: '' }]) { it(`requires explicit false to detach telemetry: ${JSON.stringify(options)}`, async () => { const context = makeContext(); @@ -257,12 +290,20 @@ describe('background telemetry close', () => { unhandled.push(reason); }; process.on('unhandledRejection', onRejection); - sinon - .stub(context.getAuthProvider()!, 'authenticate') - .callsFake(() => new Promise((resolve) => setTimeout(() => resolve({ Authorization: 'Bearer mock' }), 50))); sinon.stub(context, 'getConnectionProvider').rejects(new Error('transport unavailable')); + context.configOverrides.telemetryBatchSize = 1; const client = provider.getOrCreateClient(context, host); - addConnection(client); + const exporter = client.getExporter() as any; + const sendRequest = exporter.sendRequest.bind(exporter); + sinon + .stub(exporter, 'sendRequest') + .callsFake((_url: unknown, init: unknown) => sendRequest('http://127.0.0.1', init)); + (exporter.sendRequest as sinon.SinonStub) + .onFirstCall() + .callsFake(() => new Promise((resolve) => setTimeout(() => resolve({ ok: true, text: async () => '' }), 50))); + addConnection(client, 'in-flight'); + await clock.tickAsync(0); + addConnection(client, 'queued'); try { await provider.releaseClient(context, host, false); await clock.tickAsync(25); @@ -282,7 +323,11 @@ describe('background close built-in authentication', () => { let server: http.Server; let endpoint: string; let client: DBSQLClient; - let requests: Array<{ path?: string; authorization?: string; body: string }>; + let requests: Array<{ path?: string; authorization?: string; body: string; headers: http.IncomingHttpHeaders }>; + let firstExportStatus: number; + let firstExportDelay: number; + let firstReceived: () => void; + let firstRequest: Promise; let finishAuth: () => void; let authenticated: Promise; const refreshedToken = createAccessToken(Math.floor(Date.now() / 1000) + 3600); @@ -295,19 +340,34 @@ describe('background close built-in authentication', () => { finishAuth = resolve; }); requests = []; + firstExportStatus = 200; + firstExportDelay = 0; + firstRequest = new Promise((resolve) => { + firstReceived = resolve; + }); server = http.createServer((request, response) => { let body = ''; request.on('data', (chunk) => { body += chunk; }); request.on('end', () => { - requests.push({ path: request.url, authorization: request.headers.authorization, body }); + requests.push({ + path: request.url, + authorization: request.headers.authorization, + body, + headers: request.headers, + }); response.setHeader('Content-Type', 'application/json'); - response.end( + const result = request.url === '/oidc/v1/token' ? JSON.stringify({ access_token: refreshedToken, refresh_token: 'next-refresh', token_type: 'Bearer' }) - : '{}', - ); + : '{}'; + if (requests.length === 1) { + response.statusCode = firstExportStatus; + if (firstExportDelay) setTimeout(() => response.end(result), firstExportDelay); + else response.end(result); + firstReceived(); + } else response.end(result); }); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); @@ -339,15 +399,103 @@ describe('background close built-in authentication', () => { timestamp: Date.now(), sessionId: 'built-in-auth', }); - await client.close({ waitForTelemetry: false }); - expect((client as any).connectionProvider).to.equal(undefined); + const config = client.getConfig(); + const transport = await client.getConnectionProvider(); + let closed = false; + const closing = client.close({ waitForTelemetry: false }).then(() => { + closed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(closed).to.equal(false); + expect(await client.getConnectionProvider()).to.equal(transport); + expect(shared.getConfig()).to.equal(config); + expect(exporter.abortController).to.equal(undefined); + expect((TelemetryClientProvider.getInstance() as any).pendingCloses.size).to.equal(0); finishAuth(); - await Promise.all([...(TelemetryClientProvider.getInstance() as any).pendingCloses]); + await closing; + expect((client as any).connectionProvider).to.equal(undefined); expect(requests.map((request) => request.path)).to.deep.equal(['/oidc/v1/token', '/telemetry-ext']); expect(requests[1].authorization).to.equal(`Bearer ${refreshedToken}`); } - it('exchanges a foreign JWT after close instead of exporting the original token', async () => { + for (const nextHost of [host, 'other-close-auth.cloud.databricks.com']) { + for (const mode of ['retry', 'second-drain']) { + it(`keeps detached config across ${mode} and reconnect to ${nextHost}`, async () => { + firstExportStatus = mode === 'retry' ? 503 : 200; + firstExportDelay = mode === 'second-drain' ? 30 : 0; + await client.connect({ + host, + path: '/sql/1.0/warehouses/a?o=1111', + token: 'token-A', + customHeaders: { 'x-tenant': 'A' }, + telemetryAuthenticatedExport: true, + telemetryBatchSize: mode === 'second-drain' ? 1 : 100, + telemetryMaxRetries: 1, + telemetryCloseTimeoutMs: 1000, + }); + Object.assign(client.getConfig(), { + telemetryBackoffBaseMs: 20, + telemetryBackoffMaxMs: 20, + telemetryBackoffJitterMs: 0, + }); + const originalConfig = client.getConfig(); + const originalHeaders = originalConfig.customHeaders!; + const shared = (client as any).telemetryClient as TelemetryClient; + const exporter = shared.getExporter() as any; + const sendRequest = exporter.sendRequest.bind(exporter); + const destinations: string[] = []; + sinon.stub(exporter, 'sendRequest').callsFake((url: unknown, init: unknown) => { + destinations.push(String(url)); + return sendRequest(`${endpoint}${new URL(String(url)).pathname}`, init); + }); + const addEvent = (sessionId: string): void => + shared.getAggregator().processEvent({ + eventType: TelemetryEventType.CONNECTION_OPEN, + timestamp: Date.now(), + sessionId, + }); + addEvent('workspace-a-1'); + if (mode === 'second-drain') { + await firstRequest; + addEvent('workspace-a-2'); + } + await client.close({ waitForTelemetry: false }); + await firstRequest; + await client.connect({ + host: nextHost, + path: '/sql/1.0/warehouses/b?o=2222', + token: 'token-B', + customHeaders: { 'x-tenant': 'B' }, + telemetryAuthenticatedExport: false, + telemetryMaxRetries: 0, + telemetryCloseTimeoutMs: 1, + }); + originalHeaders['x-tenant'] = 'mutated'; + originalHeaders['x-databricks-org-id'] = '9999'; + await Promise.all([...(TelemetryClientProvider.getInstance() as any).pendingCloses]); + await client.close(); + expect(requests).to.have.length(2); + expect(destinations).to.deep.equal([`https://${host}/telemetry-ext`, `https://${host}/telemetry-ext`]); + for (const request of requests) { + expect(request.path).to.equal('/telemetry-ext'); + expect(request.authorization).to.equal('Bearer token-A'); + expect(request.headers['x-tenant']).to.equal('A'); + expect(request.headers['x-databricks-org-id']).to.equal('1111'); + } + const sessions = requests.map( + (request) => JSON.parse(JSON.parse(request.body).protoLogs[0]).entry.sql_driver_log.session_id, + ); + expect(sessions).to.deep.equal(['workspace-a-1', mode === 'retry' ? 'workspace-a-1' : 'workspace-a-2']); + expect(shared.getConfig()).not.to.equal(originalConfig); + expect(shared.getConfig().customHeaders).not.to.equal(originalHeaders); + expect(shared.getConfig().telemetryAuthenticatedExport).to.equal(true); + expect(shared.getConfig().telemetryMaxRetries).to.equal(1); + expect(shared.getConfig().telemetryCloseTimeoutMs).to.equal(1000); + }); + } + } + + it('awaits federation before close returns instead of detaching a live exchange', async () => { const getToken = FederationProvider.prototype.getToken; sinon .stub(FederationProvider.prototype, 'getToken') @@ -376,7 +524,7 @@ describe('background close built-in authentication', () => { }); for (const oauthClientSecret of [undefined, 'mock-secret']) { - it(`refreshes expired OAuth ${oauthClientSecret ? 'M2M' : 'U2M'} after close`, async () => { + it(`awaits expired OAuth ${oauthClientSecret ? 'M2M' : 'U2M'} refresh before close returns`, async () => { sinon .stub(Issuer, 'discover') .resolves(new Issuer({ issuer: `https://${host}/oidc`, token_endpoint: `${endpoint}/oidc/v1/token` })); @@ -404,9 +552,9 @@ describe('background close built-in authentication', () => { describe('background close HTTP transport', () => { it('uses the detached transport and credentials after the owning client is cleared', async () => { - let finishAuth!: () => void; - const authenticated = new Promise((resolve) => { - finishAuth = resolve; + let finishTransport!: () => void; + const transportReady = new Promise((resolve) => { + finishTransport = resolve; }); let received!: (value: { authorization?: string; body: string }) => void; const exported = new Promise<{ authorization?: string; body: string }>((resolve) => { @@ -429,22 +577,16 @@ describe('background close HTTP transport', () => { const agent = sinon.spy(context.connectionProvider, 'getAgent'); Object.assign(client as any, { host: 'detached.cloud.databricks.com', - authProvider: { - authenticate: async () => { - await authenticated; - return { Authorization: 'Bearer before-close' }; - }, - }, + authProvider: new PlainHttpAuthentication({ context, password: 'before-close' }), connectionProvider: context.connectionProvider, }); const shared = provider.getOrCreateClient(client, 'detached.cloud.databricks.com'); const exporter = shared.getExporter() as any; const sendRequest = exporter.sendRequest.bind(exporter); - const stub = sinon - .stub(exporter, 'sendRequest') - .callsFake((_url: unknown, init: unknown) => - sendRequest(`http://127.0.0.1:${(server.address() as AddressInfo).port}/telemetry-ext`, init), - ); + const stub = sinon.stub(exporter, 'sendRequest').callsFake(async (_url: unknown, init: unknown) => { + await transportReady; + return sendRequest(`http://127.0.0.1:${(server.address() as AddressInfo).port}/telemetry-ext`, init); + }); shared.getAggregator().processEvent({ eventType: TelemetryEventType.CONNECTION_OPEN, sessionId: 'detached-session', @@ -453,7 +595,7 @@ describe('background close HTTP transport', () => { try { await provider.releaseClient(client, 'detached.cloud.databricks.com', false); Object.assign(client as any, { authProvider: undefined, connectionProvider: undefined }); - finishAuth(); + finishTransport(); const request = await exported; expect(request.authorization).to.equal('Bearer before-close'); const body = JSON.parse(request.body); @@ -464,29 +606,30 @@ describe('background close HTTP transport', () => { } finally { stub.restore(); agent.restore(); - finishAuth(); + finishTransport(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); } }); - it('does not start a request when authentication finishes after the close deadline', async () => { + it('does not start a request when transport resolves after the close deadline', async () => { let requests = 0; const server = http.createServer((_request, response) => { requests += 1; response.end('{}'); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - let finishAuth!: () => void; - const authenticated = new Promise((resolve) => { - finishAuth = resolve; + let finishTransport!: () => void; + const transportReady = new Promise((resolve) => { + finishTransport = resolve; }); const context = new ClientContextStub({ telemetryCloseTimeoutMs: 10 }); - const auth = sinon.stub(context, 'getAuthProvider').returns({ - authenticate: async () => { - await authenticated; - return { Authorization: 'Bearer late' }; - }, + const auth = sinon + .stub(context, 'getAuthProvider') + .returns(new PlainHttpAuthentication({ context, password: 'late' })); + const transport = sinon.stub(context, 'getConnectionProvider').callsFake(async () => { + await transportReady; + return context.connectionProvider; }); const provider = new TelemetryClientProvider(); const shared = provider.getOrCreateClient(context, 'late.cloud.databricks.com'); @@ -501,13 +644,14 @@ describe('background close HTTP transport', () => { try { await provider.releaseClient(context, 'late.cloud.databricks.com', false); await Promise.all([...(provider as any).pendingCloses]); - finishAuth(); + finishTransport(); await new Promise((resolve) => setTimeout(resolve, 20)); expect(requests).to.equal(0); } finally { - finishAuth(); + finishTransport(); stub.restore(); auth.restore(); + transport.restore(); server.closeAllConnections(); await new Promise((resolve) => server.close(() => resolve())); }