From 5f7a754577064362baf138ee74b7302b1774bd49 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 07:28:37 +0000 Subject: [PATCH 1/5] Reuse successful feature-flag responses after access-token clients close Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 + lib/DBSQLClient.ts | 6 +- lib/FeatureFlagCache.ts | 104 +++++- tests/unit/DBSQLClient.test.ts | 17 +- .../FeatureFlagCache.retention.test.ts | 295 ++++++++++++++++++ tests/unit/telemetry/FeatureFlagCache.test.ts | 13 +- 6 files changed, 406 insertions(+), 31 deletions(-) create mode 100644 tests/unit/telemetry/FeatureFlagCache.retention.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 938983a3..d60375a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +Reuse successful Thrift feature-flag responses across access-token connections with the same host, headers, and transport settings until the original TTL expires. Idle responses are bounded and do not retain connections or timers. + - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index efc150d9..91a04d22 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -664,9 +664,9 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I } // Also usable after driver auth/transport setup, before selecting a backend. - private getFeatureFlagCache(): FeatureFlagCache { + private getFeatureFlagCache(options?: ConnectionOptions, authProvider?: IAuthentication): FeatureFlagCache { if (!this.featureFlagCache) { - this.featureFlagCache = new FeatureFlagCache(this); + this.featureFlagCache = new FeatureFlagCache(this, authProvider, options); this.featureFlagCache.getOrCreateContext(this.host!); } return this.featureFlagCache; @@ -787,7 +787,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I const internalOptions = options as ConnectionOptions & InternalConnectionOptions; const useKernel = internalOptions.useKernel === true; if (!useKernel) { - this.getFeatureFlagCache(); + this.getFeatureFlagCache(options, authProvider); } const backend = useKernel ? new KernelBackend({ context: this }) diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index 8e7f323c..73a4aa9a 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -14,63 +14,120 @@ * limitations under the License. */ -import fetch, { RequestInit, Response, Request } from 'node-fetch'; +import { createHash } from 'crypto'; +import fetch, { Headers, RequestInit, Response, Request } from 'node-fetch'; import IClientContext from './contracts/IClientContext'; import { LogLevel } from './contracts/IDBSQLLogger'; import IAuthentication from './connection/contracts/IAuthentication'; import { buildTelemetryUrl, normalizeHeaders } from './telemetry/telemetryUtils'; import buildUserAgentString from './utils/buildUserAgentString'; import driverVersion from './version'; +import { ConnectionOptions } from './contracts/IDBSQLClient'; export interface FeatureFlagContext { flags?: Map; fetchPromise?: Promise; - lastFetched?: Date; + lastFetched?: number; + cacheable?: boolean; refCount: number; cacheDuration: number; } /** - * Shared feature-flag values per workspace. + * Shared feature-flag values per request identity. * Acquire/release a context per consumer; each reader uses its caller's auth. * Responsibilities: * - dedupe in-flight fetches (thundering-herd protection); - * - ref-count so context goes away when the last consumer closes; + * - retain bounded successful responses until their original expiry; * - clamp server-provided TTL into a safe band. * - * Workspace ID partitions SPOG traffic; normalized host is the fallback. + * Workspace routing, authentication, and transport settings partition responses. */ export default class FeatureFlagCache { private static sharedContexts = new Map(); + private static idleContexts = new Map(); + + private static authenticationIds = new WeakMap(); + + private static nextAuthenticationId = 0; + + private static readonly MAX_IDLE_CONTEXTS = 100; + private contexts = FeatureFlagCache.sharedContexts; private readonly userAgent: string; + private readonly scope: string; + + private readonly retainIdle: boolean; + private readonly CACHE_DURATION_MS = 15 * 60 * 1000; private readonly MIN_CACHE_DURATION_S = 60; private readonly MAX_CACHE_DURATION_S = 3600; - constructor(private context: IClientContext, private authProvider?: IAuthentication) { + constructor( + private context: IClientContext, + private authProvider?: IAuthentication, + connectionOptions?: ConnectionOptions, + ) { this.userAgent = buildUserAgentString(this.context.getConfig().userAgentEntry); + const authentication = this.authProvider ?? this.context.getAuthProvider?.(); + const tokenAuth = + !authProvider && + connectionOptions !== undefined && + (connectionOptions.authType === undefined || connectionOptions.authType === 'access-token'); + let authenticationId: number | undefined; + if (authentication && !tokenAuth) { + authenticationId = FeatureFlagCache.authenticationIds.get(authentication); + if (authenticationId === undefined) { + authenticationId = FeatureFlagCache.nextAuthenticationId; + FeatureFlagCache.nextAuthenticationId += 1; + FeatureFlagCache.authenticationIds.set(authentication, authenticationId); + } + } + this.retainIdle = tokenAuth || !authentication; + this.scope = createHash('sha256') + .update( + JSON.stringify([ + this.userAgent, + [...new Headers(this.context.getConfig().customHeaders ?? {}).entries()], + tokenAuth ? connectionOptions?.token ?? 'anonymous' : authenticationId, + connectionOptions?.https, + connectionOptions?.port, + connectionOptions?.proxy, + connectionOptions?.customCaCert, + connectionOptions?.clientCert, + connectionOptions?.clientKey, + connectionOptions?.checkServerCertificate, + ]), + ) + .digest('hex'); } private cacheKey(host: string): string { - const headers = this.context.getConfig().customHeaders ?? {}; - const workspace = Object.entries(headers).find(([name]) => name.toLowerCase() === 'x-databricks-org-id')?.[1]; - return workspace ? `workspace:${workspace}` : `host:${buildTelemetryUrl(host, '') ?? host}`; + return JSON.stringify([buildTelemetryUrl(host, '') ?? host, this.scope]); + } + + private isExpired(ctx: FeatureFlagContext): boolean { + return ctx.lastFetched === undefined || performance.now() - ctx.lastFetched > ctx.cacheDuration; } getOrCreateContext(host: string): FeatureFlagContext { const key = this.cacheKey(host); let ctx = this.contexts.get(key); if (!ctx) { - ctx = { - refCount: 0, - cacheDuration: this.CACHE_DURATION_MS, - }; + const idle = FeatureFlagCache.idleContexts.get(key); + FeatureFlagCache.idleContexts.delete(key); + ctx = + idle && !this.isExpired(idle) + ? idle + : { + refCount: 0, + cacheDuration: this.CACHE_DURATION_MS, + }; this.contexts.set(key, ctx); } ctx.refCount += 1; @@ -84,6 +141,17 @@ export default class FeatureFlagCache { ctx.refCount -= 1; if (ctx.refCount <= 0) { this.contexts.delete(key); + if (this.retainIdle && ctx.cacheable && !ctx.fetchPromise && !this.isExpired(ctx)) { + const idle = FeatureFlagCache.idleContexts; + for (const [idleKey, value] of idle) { + if (this.isExpired(value)) idle.delete(idleKey); + } + idle.delete(key); + idle.set(key, ctx); + while (idle.size > FeatureFlagCache.MAX_IDLE_CONTEXTS) { + idle.delete(idle.keys().next().value!); + } + } } } } @@ -96,14 +164,14 @@ export default class FeatureFlagCache { return undefined; } - const isExpired = !ctx.lastFetched || Date.now() - ctx.lastFetched.getTime() > ctx.cacheDuration; + const isExpired = this.isExpired(ctx); if (isExpired) { if (!ctx.fetchPromise) { ctx.fetchPromise = this.fetchFeatureFlags(host) .then((flags) => { ctx.flags = flags; - ctx.lastFetched = new Date(); + ctx.lastFetched = performance.now(); }) .catch((error: any) => { logger.log(LogLevel.debug, `Error fetching feature flag: ${error.message}`); @@ -175,6 +243,7 @@ export default class FeatureFlagCache { private async fetchFeatureFlags(host: string): Promise> { const logger = this.context.getLogger(); const ctx = this.contexts.get(this.cacheKey(host)); + if (ctx) ctx.cacheable = false; try { const endpoint = buildTelemetryUrl( @@ -209,6 +278,11 @@ export default class FeatureFlagCache { const data: any = await response.json(); if (data && data.flags && Array.isArray(data.flags)) { + if (ctx) { + ctx.cacheable = data.flags.every( + (flag: any) => typeof flag?.name === 'string' && typeof flag.value === 'string', + ); + } if (ctx && Number.isFinite(data.ttl_seconds) && data.ttl_seconds > 0) { const clampedTtl = Math.max(this.MIN_CACHE_DURATION_S, Math.min(this.MAX_CACHE_DURATION_S, data.ttl_seconds)); ctx.cacheDuration = clampedTtl * 1000; diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index 497889db..0ec311c7 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -1094,7 +1094,10 @@ describe('DBSQLClient feature flag lifecycle', () => { }; const contexts = (FeatureFlagCache as any).sharedContexts as Map; - beforeEach(() => contexts.clear()); + beforeEach(() => { + contexts.clear(); + (FeatureFlagCache as any).idleContexts.clear(); + }); afterEach(() => { sinon.restore(); contexts.clear(); @@ -1107,7 +1110,7 @@ describe('DBSQLClient feature flag lifecycle', () => { .resolves({ Authorization: 'Bearer oauth-token' }); const fetch = sinon .stub(FeatureFlagCache.prototype as any, 'fetchWithRetry') - .resolves(new Response(JSON.stringify({ flags: [{ name: 'sampleLimit', value: '42' }] }))); + .callsFake(async () => new Response(JSON.stringify({ flags: [{ name: 'sampleLimit', value: '42' }] }))); const connect = sinon.stub(ThriftBackend.prototype, 'connect').resolves(); sinon.stub(ThriftBackend.prototype, 'close').resolves(); const clients = [new DBSQLClient(), new DBSQLClient()]; @@ -1126,15 +1129,15 @@ describe('DBSQLClient feature flag lifecycle', () => { }), ); expect(connect.callCount).to.equal(2); - expect(fetch.callCount).to.equal(1); + expect(fetch.callCount).to.equal(oauth ? 2 : 1); const { headers } = fetch.firstCall.args[1]; expect(headers.Authorization).to.equal(oauth ? 'Bearer oauth-token' : `Bearer ${options.token}`); expect(headers['x-databricks-org-id']).to.equal('123'); - expect(auth.callCount).to.equal(oauth ? 1 : 0); + expect(auth.callCount).to.equal(oauth ? 2 : 0); await clients[0].close(); - expect(contexts.has('workspace:123')).to.be.true; + expect(contexts.size).to.equal(1); await clients[1].close(); - expect(contexts.has('workspace:123')).to.be.false; + expect(contexts.size).to.equal(0); }); } @@ -1164,7 +1167,7 @@ describe('DBSQLClient feature flag lifecycle', () => { } catch (caught) { expect(caught).to.equal(error); } - expect(contexts.has('workspace:123')).to.be.false; + expect(contexts.size).to.equal(0); }); } }); diff --git a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts new file mode 100644 index 00000000..832f2e90 --- /dev/null +++ b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts @@ -0,0 +1,295 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { Response } from 'node-fetch'; +import FeatureFlagCache from '../../../lib/FeatureFlagCache'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; +import DBSQLClient from '../../../lib/DBSQLClient'; +import ThriftBackend from '../../../lib/thrift-backend/ThriftBackend'; +import TelemetryClientProvider from '../../../lib/telemetry/TelemetryClientProvider'; + +describe('FeatureFlagCache idle retention', () => { + const host = 'cache.databricks.com'; + const options = { host, path: '/sql/1.0/warehouses/test', token: 'identity-a' }; + let clock: sinon.SinonFakeTimers; + + beforeEach(() => { + (FeatureFlagCache as any).sharedContexts.clear(); + (FeatureFlagCache as any).idleContexts?.clear(); + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + sinon.restore(); + TelemetryClientProvider.__resetInstanceForTests(); + clock.restore(); + }); + + function makeCache(overrides: Partial = {}, config: any = {}, auth?: any) { + return new FeatureFlagCache(new ClientContextStub(config), auth, { ...options, ...overrides } as ConnectionOptions); + } + + function response(value = 'true', ttl = 60) { + return new Response(JSON.stringify({ flags: [{ name: 'flag', value }], ttl_seconds: ttl })); + } + + it('reuses a successful response after the last consumer closes', async () => { + const first = makeCache(); + const fetch = sinon.stub(first as any, 'fetchWithRetry').resolves(response()); + first.getOrCreateContext(host); + expect(await first.getBoolean(host, 'flag')).to.equal(true); + first.releaseContext(host); + const next = makeCache(); + const nextFetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response('false')); + next.getOrCreateContext(host); + expect(await next.getBoolean(host, 'flag')).to.equal(true); + expect(fetch.callCount).to.equal(1); + expect(nextFetch.callCount).to.equal(0); + next.releaseContext(host); + }); + + it('keeps the original expiry across repeated close/reopen and wall-clock jumps', async () => { + const first = makeCache(); + sinon.stub(first as any, 'fetchWithRetry').resolves(response()); + first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); + first.releaseContext(host); + clock.tick(59000); + clock.setSystemTime(-100000000); + const next = makeCache(); + const fetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response('false')); + next.getOrCreateContext(host); + expect(await next.getBoolean(host, 'flag')).to.equal(true); + clock.tick(1001); + expect(await next.getBoolean(host, 'flag')).to.equal(false); + expect(fetch.callCount).to.equal(1); + next.releaseContext(host); + }); + + it('does not expire early on a forward wall-clock jump', async () => { + const cache = makeCache(); + const fetch = sinon.stub(cache as any, 'fetchWithRetry').resolves(response()); + cache.getOrCreateContext(host); + await cache.getBoolean(host, 'flag'); + clock.setSystemTime(100000000); + await cache.getBoolean(host, 'flag'); + expect(fetch.callCount).to.equal(1); + cache.releaseContext(host); + }); + + it('preserves every typed reader and null/missing defaults after close', async () => { + const cases: Array<[string, string, any]> = [ + ['getBoolean', 'True', true], + ['getBoolean', 'FALSE', false], + ['getBoolean', 'null', false], + ['getInt32', '-2147483648', -2147483648], + ['getInt32', '2147483648', undefined], + ['getInt64', '9223372036854775807', BigInt('9223372036854775807')], + ['getInt64', '9223372036854775808', undefined], + ['getDouble', '1.25', 1.25], + ['getDouble', '1e400', undefined], + ['getString', '"hello"', 'hello'], + ['getString', 'null', undefined], + ['getStringList', '["a","b"]', ['a', 'b']], + ['getStringList', '[null]', undefined], + ]; + const fetch = sinon + .stub(FeatureFlagCache.prototype as any, 'fetchWithRetry') + .resolves( + new Response(JSON.stringify({ flags: cases.map(([, value], index) => ({ name: String(index), value })) })), + ); + for (let iteration = 0; iteration < 2; iteration += 1) { + const cache = makeCache(); + cache.getOrCreateContext(host); + for (const [index, [method, , expected]] of cases.entries()) { + expect(await (cache as any)[method](host, String(index))).to.deep.equal(expected); + } + expect(await cache.getString(host, 'missing', 'fallback')).to.equal('fallback'); + cache.releaseContext(host); + } + expect(fetch.callCount).to.equal(1); + }); + + for (const enabled of [true, false]) { + it(`preserves telemetry enablement (${enabled}) and the environment kill switch on an idle hit`, async () => { + sinon.stub(ThriftBackend.prototype, 'connect').resolves(); + sinon.stub(ThriftBackend.prototype, 'close').resolves(); + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake( + async () => + new Response( + JSON.stringify({ + flags: [ + { + name: 'databricks.partnerplatform.clientConfigsFeatureFlags.enableTelemetryForNodeJs', + value: String(enabled), + }, + ], + }), + ), + ); + const previous = process.env.DATABRICKS_TELEMETRY_DISABLED; + delete process.env.DATABRICKS_TELEMETRY_DISABLED; + try { + for (const killed of [false, true, false]) { + if (killed) process.env.DATABRICKS_TELEMETRY_DISABLED = 'true'; + else delete process.env.DATABRICKS_TELEMETRY_DISABLED; + const client = new DBSQLClient({ logger: { log: () => {} } }); + await client.connect(options); + expect(client.getTelemetryStats() !== undefined).to.equal(enabled && !killed); + await client.close(); + } + expect(fetch.callCount).to.equal(1); + } finally { + if (previous === undefined) delete process.env.DATABRICKS_TELEMETRY_DISABLED; + else process.env.DATABRICKS_TELEMETRY_DISABLED = previous; + } + }); + } + + for (const [name, overrides, config] of [ + ['token', { token: 'identity-b' }, {}], + ['workspace', {}, { customHeaders: { 'x-databricks-org-id': '2' } }], + ['user agent', {}, { userAgentEntry: 'other-app' }], + ['custom header', {}, { customHeaders: { 'X-Custom': 'other' } }], + ['server trust', { checkServerCertificate: false }, {}], + ['custom CA', { customCaCert: 'ca-other' }, {}], + ['client certificate', { clientCert: 'cert-other', clientKey: 'key-other' }, {}], + ['proxy', { proxy: { protocol: 'http', host: 'proxy.test', port: 8080 } }, {}], + ['protocol', { https: false }, {}], + ['port', { port: 8443 }, {}], + ] as Array<[string, Partial, any]>) { + it(`partitions active and idle responses by ${name}`, async () => { + const first = makeCache(); + sinon.stub(first as any, 'fetchWithRetry').resolves(response()); + const original = first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); + const other = makeCache(overrides, config); + expect(other.getOrCreateContext(host)).to.not.equal(original); + other.releaseContext(host); + first.releaseContext(host); + const next = makeCache(overrides, config); + const fetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response('false')); + next.getOrCreateContext(host); + expect(await next.getBoolean(host, 'flag')).to.equal(false); + expect(fetch.callCount).to.equal(1); + next.releaseContext(host); + }); + } + + it('includes the host even when the workspace is the same', () => { + const cache = makeCache({}, { customHeaders: { 'x-databricks-org-id': '1' } }); + expect(cache.getOrCreateContext(host)).to.not.equal(cache.getOrCreateContext('other.databricks.com')); + cache.releaseContext(host); + cache.releaseContext('other.databricks.com'); + }); + + it('canonicalizes host and custom header casing and order', () => { + const first = makeCache({}, { customHeaders: { 'X-Route': 'a', 'X-Databricks-Org-Id': '1' } }); + const next = makeCache({}, { customHeaders: { 'x-databricks-org-id': '1', 'x-route': 'a' } }); + expect(first.getOrCreateContext(host)).to.equal(next.getOrCreateContext(`https://${host.toUpperCase()}:443/`)); + first.releaseContext(host); + next.releaseContext(host); + }); + + it('never retains a dynamic or deprecated authentication provider across close', async () => { + const auth = { authenticate: sinon.stub().resolves({ Authorization: 'Bearer changing-token' }) }; + for (let iteration = 0; iteration < 2; iteration += 1) { + const cache = makeCache({}, {}, auth); + const fetch = sinon.stub(cache as any, 'fetchWithRetry').resolves(response(iteration === 0 ? 'true' : 'false')); + cache.getOrCreateContext(host); + expect(await cache.getBoolean(host, 'flag')).to.equal(iteration === 0); + expect(fetch.callCount).to.equal(1); + cache.releaseContext(host); + } + expect((FeatureFlagCache as any).idleContexts.size).to.equal(0); + expect(auth.authenticate.callCount).to.equal(2); + }); + + for (const [name, body, status] of [ + ['missing flags', {}, 200], + ['null flags', { flags: null }, 200], + ['malformed flags', { flags: [{ name: 'flag', value: true }] }, 200], + ['partially malformed flags', { flags: [{ name: 'flag', value: 'true' }, null] }, 200], + ['invalid JSON', '{', 200], + ['HTTP failure', {}, 403], + ] as Array<[string, any, number]>) { + it(`does not retain ${name} after close`, async () => { + const first = makeCache(); + const fetch = sinon + .stub(first as any, 'fetchWithRetry') + .resolves(new Response(typeof body === 'string' ? body : JSON.stringify(body), { status })); + first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); + await first.getBoolean(host, 'flag'); + expect(fetch.callCount).to.equal(1); + first.releaseContext(host); + const next = makeCache(); + const nextFetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response()); + next.getOrCreateContext(host); + expect(await next.getBoolean(host, 'flag')).to.equal(true); + expect(nextFetch.callCount).to.equal(1); + next.releaseContext(host); + }); + } + + it('deduplicates a cold concurrent burst and a burst after idle', async () => { + let complete!: (response: Response) => void; + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').returns( + new Promise((resolve) => { + complete = resolve; + }), + ); + const clients = Array.from({ length: 50 }, () => makeCache()); + clients.forEach((cache) => cache.getOrCreateContext(host)); + const reads = clients.map((cache) => cache.getBoolean(host, 'flag')); + await clock.tickAsync(0); + expect(fetch.callCount).to.equal(1); + complete(response()); + expect(await Promise.all(reads)).to.deep.equal(Array(50).fill(true)); + clients.forEach((cache) => cache.releaseContext(host)); + clock.tick(2000); + for (const cache of clients) cache.getOrCreateContext(host); + expect(await Promise.all(clients.map((cache) => cache.getBoolean(host, 'flag')))).to.deep.equal( + Array(50).fill(true), + ); + expect(fetch.callCount).to.equal(1); + clients.forEach((cache) => cache.releaseContext(host)); + }); + + it('does not retain a response completed after release', async () => { + let complete!: (response: Response) => void; + const cache = makeCache(); + sinon.stub(cache as any, 'fetchWithRetry').returns( + new Promise((resolve) => { + complete = resolve; + }), + ); + cache.getOrCreateContext(host); + const read = cache.getBoolean(host, 'flag'); + await clock.tickAsync(0); + cache.releaseContext(host); + complete(response()); + await read; + expect((FeatureFlagCache as any).sharedContexts.size).to.equal(0); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(0); + }); + + it('bounds retained contexts after 1000 distinct identities without retaining credentials', async () => { + sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); + for (let iteration = 0; iteration < 1000; iteration += 1) { + const cache = makeCache({ token: `secret-identity-${iteration}` }); + cache.getOrCreateContext(host); + expect(await cache.getBoolean(host, 'flag')).to.equal(true); + cache.releaseContext(host); + } + expect((FeatureFlagCache as any).sharedContexts.size).to.equal(0); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(100); + expect(JSON.stringify([...(FeatureFlagCache as any).idleContexts])).to.not.contain('secret-identity'); + clock.tick(61000); + const next = makeCache(); + next.getOrCreateContext(host); + await next.getBoolean(host, 'flag'); + next.releaseContext(host); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(1); + }); +}); diff --git a/tests/unit/telemetry/FeatureFlagCache.test.ts b/tests/unit/telemetry/FeatureFlagCache.test.ts index 7f1ea951..65ac21ac 100644 --- a/tests/unit/telemetry/FeatureFlagCache.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.test.ts @@ -26,6 +26,7 @@ describe('FeatureFlagCache', () => { beforeEach(() => { (FeatureFlagCache as any).sharedContexts.clear(); + (FeatureFlagCache as any).idleContexts.clear(); clock = sinon.useFakeTimers(); }); @@ -408,11 +409,11 @@ describe('FeatureFlagCache', () => { expect(fetchStub.calledOnce).to.be.true; }); - it('shares values per workspace, refreshes with the current caller, and defaults on failure', async () => { + it('shares values for the same host and workspace, refreshes with the current caller, and defaults on failure', async () => { const first = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'x-databricks-org-id': '1' } })); const current = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'X-Databricks-Org-Id': '1' } })); const other = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'x-databricks-org-id': '2' } })); - expect(first.getOrCreateContext('host-a')).to.equal(current.getOrCreateContext('host-b')); + expect(first.getOrCreateContext('host-a')).to.equal(current.getOrCreateContext('host-a')); expect(other.getOrCreateContext('host-a')).to.not.equal(first.getOrCreateContext('host-a')); const response = (value: string) => new Response( @@ -423,18 +424,18 @@ describe('FeatureFlagCache', () => { ); const firstFetch = sinon.stub(first as any, 'fetchWithRetry').resolves(response('true')); const currentFetch = sinon.stub(current as any, 'fetchWithRetry').resolves(response('FALSE')); - expect(await Promise.all([first.getBoolean('host-a', 'flag'), current.getBoolean('host-b', 'flag')])).to.deep.equal( + expect(await Promise.all([first.getBoolean('host-a', 'flag'), current.getBoolean('host-a', 'flag')])).to.deep.equal( [true, true], ); expect(currentFetch.called).to.be.false; clock.tick(61000); - expect(await current.getBoolean('host-b', 'flag', true)).to.be.false; + expect(await current.getBoolean('host-a', 'flag', true)).to.be.false; expect(firstFetch.calledOnce).to.be.true; expect(currentFetch.calledOnce).to.be.true; clock.tick(61000); currentFetch.rejects(new Error('offline')); - expect(await current.getBoolean('host-b', 'flag', true)).to.be.true; - expect(await current.getBoolean('host-b', 'flag')).to.be.false; + expect(await current.getBoolean('host-a', 'flag', true)).to.be.true; + expect(await current.getBoolean('host-a', 'flag')).to.be.false; expect(currentFetch.calledTwice).to.be.true; }); }); From 3ce075d6ccbc2d141339c87b8744e6a6f19f9d13 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 07:52:54 +0000 Subject: [PATCH 2/5] Preserve header validation and verify idle reconnect behavior Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/FeatureFlagCache.ts | 8 ++- tests/e2e/feature_flag_cache.test.ts | 56 +++++++++++++++++++ .../telemetry/telemetry-integration.test.ts | 2 + .../FeatureFlagCache.retention.test.ts | 8 +++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/feature_flag_cache.test.ts diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index 73a4aa9a..3ca1f5f9 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -15,7 +15,7 @@ */ import { createHash } from 'crypto'; -import fetch, { Headers, RequestInit, Response, Request } from 'node-fetch'; +import fetch, { RequestInit, Response, Request } from 'node-fetch'; import IClientContext from './contracts/IClientContext'; import { LogLevel } from './contracts/IDBSQLLogger'; import IAuthentication from './connection/contracts/IAuthentication'; @@ -93,8 +93,10 @@ export default class FeatureFlagCache { .update( JSON.stringify([ this.userAgent, - [...new Headers(this.context.getConfig().customHeaders ?? {}).entries()], - tokenAuth ? connectionOptions?.token ?? 'anonymous' : authenticationId, + Object.entries(this.context.getConfig().customHeaders ?? {}) + .map(([name, value]) => [name.toLowerCase(), value]) + .sort(([left], [right]) => left.localeCompare(right)), + tokenAuth ? connectionOptions?.token : authenticationId, connectionOptions?.https, connectionOptions?.port, connectionOptions?.proxy, diff --git a/tests/e2e/feature_flag_cache.test.ts b/tests/e2e/feature_flag_cache.test.ts new file mode 100644 index 00000000..d1fc64c4 --- /dev/null +++ b/tests/e2e/feature_flag_cache.test.ts @@ -0,0 +1,56 @@ +import { expect } from 'chai'; +import { channel } from 'diagnostics_channel'; +import { DBSQLClient } from '../../lib'; +import config from './utils/config'; + +describe('Thrift feature flags across idle reconnects', () => { + for (const reuseClient of [false, true]) { + it(`reuses flags and preserves results when ${ + reuseClient ? 'reconnecting a client' : 'creating another client' + }`, async () => { + let requests = 0; + const onRequest = ({ request }: any) => { + if (request.method === 'GET' && request.path.includes('/connector-service/feature-flags/NODEJS/')) { + requests += 1; + } + }; + const requestsChannel = channel('http.client.request.start'); + requestsChannel.subscribe(onRequest); + let client = new DBSQLClient(); + let enabled: boolean | undefined; + try { + const cycle = async (iteration: number) => { + if (!reuseClient && iteration > 0) client = new DBSQLClient(); + try { + await client.connect({ + host: config.host, + path: config.path, + token: config.token, + userAgentEntry: `feature-flag-reconnect-${reuseClient}`, + }); + if (iteration === 0) enabled = client.getTelemetryStats() !== undefined; + expect(client.getTelemetryStats() !== undefined).to.equal(enabled); + const session = await client.openSession(); + try { + const operation = await session.executeStatement('SELECT 1 AS value, CAST(NULL AS STRING) AS missing'); + try { + expect(await operation.fetchAll({ maxRows: 1 })).to.deep.equal([{ value: 1, missing: null }]); + } finally { + await operation.close(); + } + } finally { + await session.close(); + } + } finally { + await client.close(); + } + }; + await cycle(0); + await cycle(1); + expect(requests).to.equal(1); + } finally { + requestsChannel.unsubscribe(onRequest); + } + }); + } +}); diff --git a/tests/e2e/telemetry/telemetry-integration.test.ts b/tests/e2e/telemetry/telemetry-integration.test.ts index 6ff02cff..cd18e437 100644 --- a/tests/e2e/telemetry/telemetry-integration.test.ts +++ b/tests/e2e/telemetry/telemetry-integration.test.ts @@ -92,11 +92,13 @@ describe('Telemetry Integration', () => { beforeEach(() => { TelemetryClientProvider.__resetInstanceForTests(); flagContexts.clear(); + (FeatureFlagCache as any).idleContexts.clear(); }); afterEach(() => { TelemetryClientProvider.__resetInstanceForTests(); flagContexts.clear(); + (FeatureFlagCache as any).idleContexts.clear(); sinon.restore(); }); diff --git a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts index 832f2e90..fa6465df 100644 --- a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts @@ -183,6 +183,14 @@ describe('FeatureFlagCache idle retention', () => { cache.releaseContext('other.databricks.com'); }); + it('does not alias an absent token with a literal anonymous token', () => { + const anonymous = makeCache({ token: undefined }); + const token = makeCache({ token: 'anonymous' }); + expect(anonymous.getOrCreateContext(host)).to.not.equal(token.getOrCreateContext(host)); + anonymous.releaseContext(host); + token.releaseContext(host); + }); + it('canonicalizes host and custom header casing and order', () => { const first = makeCache({}, { customHeaders: { 'X-Route': 'a', 'X-Databricks-Org-Id': '1' } }); const next = makeCache({}, { customHeaders: { 'x-databricks-org-id': '1', 'x-route': 'a' } }); From f6abb2f96251364b821b42f4c4765a0ff861d799 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 08:35:51 +0000 Subject: [PATCH 3/5] Preserve workspace-wide feature-flag sharing across idle reconnects Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/DBSQLClient.ts | 6 +- lib/FeatureFlagCache.ts | 59 +----- tests/e2e/feature_flag_cache.test.ts | 5 + tests/unit/DBSQLClient.test.ts | 12 +- .../FeatureFlagCache.retention.test.ts | 183 ++++++++++++------ tests/unit/telemetry/FeatureFlagCache.test.ts | 12 +- 7 files changed, 150 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d60375a6..85b96d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Reuse successful Thrift feature-flag responses across access-token connections with the same host, headers, and transport settings until the original TTL expires. Idle responses are bounded and do not retain connections or timers. +Reuse successful Thrift feature-flag responses across connections to the same workspace (or host), using the existing sharing key for every auth type. Idle responses are bounded and expire at the original server TTL (currently up to one hour), as for a long-lived client; they do not retain connections or timers. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index 91a04d22..efc150d9 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -664,9 +664,9 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I } // Also usable after driver auth/transport setup, before selecting a backend. - private getFeatureFlagCache(options?: ConnectionOptions, authProvider?: IAuthentication): FeatureFlagCache { + private getFeatureFlagCache(): FeatureFlagCache { if (!this.featureFlagCache) { - this.featureFlagCache = new FeatureFlagCache(this, authProvider, options); + this.featureFlagCache = new FeatureFlagCache(this); this.featureFlagCache.getOrCreateContext(this.host!); } return this.featureFlagCache; @@ -787,7 +787,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient, I const internalOptions = options as ConnectionOptions & InternalConnectionOptions; const useKernel = internalOptions.useKernel === true; if (!useKernel) { - this.getFeatureFlagCache(options, authProvider); + this.getFeatureFlagCache(); } const backend = useKernel ? new KernelBackend({ context: this }) diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index 3ca1f5f9..66eebfb6 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { createHash } from 'crypto'; import fetch, { RequestInit, Response, Request } from 'node-fetch'; import IClientContext from './contracts/IClientContext'; import { LogLevel } from './contracts/IDBSQLLogger'; @@ -22,7 +21,6 @@ import IAuthentication from './connection/contracts/IAuthentication'; import { buildTelemetryUrl, normalizeHeaders } from './telemetry/telemetryUtils'; import buildUserAgentString from './utils/buildUserAgentString'; import driverVersion from './version'; -import { ConnectionOptions } from './contracts/IDBSQLClient'; export interface FeatureFlagContext { flags?: Map; @@ -34,83 +32,40 @@ export interface FeatureFlagContext { } /** - * Shared feature-flag values per request identity. + * Shared feature-flag values per workspace. * Acquire/release a context per consumer; each reader uses its caller's auth. * Responsibilities: * - dedupe in-flight fetches (thundering-herd protection); * - retain bounded successful responses until their original expiry; * - clamp server-provided TTL into a safe band. * - * Workspace routing, authentication, and transport settings partition responses. + * Workspace ID partitions SPOG traffic; normalized host is the fallback. */ export default class FeatureFlagCache { private static sharedContexts = new Map(); private static idleContexts = new Map(); - private static authenticationIds = new WeakMap(); - - private static nextAuthenticationId = 0; - private static readonly MAX_IDLE_CONTEXTS = 100; private contexts = FeatureFlagCache.sharedContexts; private readonly userAgent: string; - private readonly scope: string; - - private readonly retainIdle: boolean; - private readonly CACHE_DURATION_MS = 15 * 60 * 1000; private readonly MIN_CACHE_DURATION_S = 60; private readonly MAX_CACHE_DURATION_S = 3600; - constructor( - private context: IClientContext, - private authProvider?: IAuthentication, - connectionOptions?: ConnectionOptions, - ) { + constructor(private context: IClientContext, private authProvider?: IAuthentication) { this.userAgent = buildUserAgentString(this.context.getConfig().userAgentEntry); - const authentication = this.authProvider ?? this.context.getAuthProvider?.(); - const tokenAuth = - !authProvider && - connectionOptions !== undefined && - (connectionOptions.authType === undefined || connectionOptions.authType === 'access-token'); - let authenticationId: number | undefined; - if (authentication && !tokenAuth) { - authenticationId = FeatureFlagCache.authenticationIds.get(authentication); - if (authenticationId === undefined) { - authenticationId = FeatureFlagCache.nextAuthenticationId; - FeatureFlagCache.nextAuthenticationId += 1; - FeatureFlagCache.authenticationIds.set(authentication, authenticationId); - } - } - this.retainIdle = tokenAuth || !authentication; - this.scope = createHash('sha256') - .update( - JSON.stringify([ - this.userAgent, - Object.entries(this.context.getConfig().customHeaders ?? {}) - .map(([name, value]) => [name.toLowerCase(), value]) - .sort(([left], [right]) => left.localeCompare(right)), - tokenAuth ? connectionOptions?.token : authenticationId, - connectionOptions?.https, - connectionOptions?.port, - connectionOptions?.proxy, - connectionOptions?.customCaCert, - connectionOptions?.clientCert, - connectionOptions?.clientKey, - connectionOptions?.checkServerCertificate, - ]), - ) - .digest('hex'); } private cacheKey(host: string): string { - return JSON.stringify([buildTelemetryUrl(host, '') ?? host, this.scope]); + const headers = this.context.getConfig().customHeaders ?? {}; + const workspace = Object.entries(headers).find(([name]) => name.toLowerCase() === 'x-databricks-org-id')?.[1]; + return workspace ? `workspace:${workspace}` : `host:${buildTelemetryUrl(host, '') ?? host}`; } private isExpired(ctx: FeatureFlagContext): boolean { @@ -143,7 +98,7 @@ export default class FeatureFlagCache { ctx.refCount -= 1; if (ctx.refCount <= 0) { this.contexts.delete(key); - if (this.retainIdle && ctx.cacheable && !ctx.fetchPromise && !this.isExpired(ctx)) { + if (ctx.cacheable && !ctx.fetchPromise && !this.isExpired(ctx)) { const idle = FeatureFlagCache.idleContexts; for (const [idleKey, value] of idle) { if (this.isExpired(value)) idle.delete(idleKey); diff --git a/tests/e2e/feature_flag_cache.test.ts b/tests/e2e/feature_flag_cache.test.ts index d1fc64c4..90892b4c 100644 --- a/tests/e2e/feature_flag_cache.test.ts +++ b/tests/e2e/feature_flag_cache.test.ts @@ -2,8 +2,13 @@ import { expect } from 'chai'; import { channel } from 'diagnostics_channel'; import { DBSQLClient } from '../../lib'; import config from './utils/config'; +import FeatureFlagCache from '../../lib/FeatureFlagCache'; describe('Thrift feature flags across idle reconnects', () => { + beforeEach(() => { + (FeatureFlagCache as any).idleContexts.clear(); + }); + for (const reuseClient of [false, true]) { it(`reuses flags and preserves results when ${ reuseClient ? 'reconnecting a client' : 'creating another client' diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index 0ec311c7..ef62caaf 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -1110,7 +1110,7 @@ describe('DBSQLClient feature flag lifecycle', () => { .resolves({ Authorization: 'Bearer oauth-token' }); const fetch = sinon .stub(FeatureFlagCache.prototype as any, 'fetchWithRetry') - .callsFake(async () => new Response(JSON.stringify({ flags: [{ name: 'sampleLimit', value: '42' }] }))); + .resolves(new Response(JSON.stringify({ flags: [{ name: 'sampleLimit', value: '42' }] }))); const connect = sinon.stub(ThriftBackend.prototype, 'connect').resolves(); sinon.stub(ThriftBackend.prototype, 'close').resolves(); const clients = [new DBSQLClient(), new DBSQLClient()]; @@ -1129,15 +1129,15 @@ describe('DBSQLClient feature flag lifecycle', () => { }), ); expect(connect.callCount).to.equal(2); - expect(fetch.callCount).to.equal(oauth ? 2 : 1); + expect(fetch.callCount).to.equal(1); const { headers } = fetch.firstCall.args[1]; expect(headers.Authorization).to.equal(oauth ? 'Bearer oauth-token' : `Bearer ${options.token}`); expect(headers['x-databricks-org-id']).to.equal('123'); - expect(auth.callCount).to.equal(oauth ? 2 : 0); + expect(auth.callCount).to.equal(oauth ? 1 : 0); await clients[0].close(); - expect(contexts.size).to.equal(1); + expect(contexts.has('workspace:123')).to.be.true; await clients[1].close(); - expect(contexts.size).to.equal(0); + expect(contexts.has('workspace:123')).to.be.false; }); } @@ -1167,7 +1167,7 @@ describe('DBSQLClient feature flag lifecycle', () => { } catch (caught) { expect(caught).to.equal(error); } - expect(contexts.size).to.equal(0); + expect(contexts.has('workspace:123')).to.be.false; }); } }); diff --git a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts index fa6465df..0216071a 100644 --- a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts @@ -7,6 +7,8 @@ import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; import DBSQLClient from '../../../lib/DBSQLClient'; import ThriftBackend from '../../../lib/thrift-backend/ThriftBackend'; import TelemetryClientProvider from '../../../lib/telemetry/TelemetryClientProvider'; +import DatabricksOAuth from '../../../lib/connection/auth/DatabricksOAuth'; +import StaticTokenProvider from '../../../lib/connection/auth/tokenProvider/StaticTokenProvider'; describe('FeatureFlagCache idle retention', () => { const host = 'cache.databricks.com'; @@ -25,8 +27,8 @@ describe('FeatureFlagCache idle retention', () => { clock.restore(); }); - function makeCache(overrides: Partial = {}, config: any = {}, auth?: any) { - return new FeatureFlagCache(new ClientContextStub(config), auth, { ...options, ...overrides } as ConnectionOptions); + function makeCache(config: any = {}, auth?: any) { + return new FeatureFlagCache(new ClientContextStub(config), auth); } function response(value = 'true', ttl = 60) { @@ -146,72 +148,126 @@ describe('FeatureFlagCache idle retention', () => { }); } - for (const [name, overrides, config] of [ - ['token', { token: 'identity-b' }, {}], - ['workspace', {}, { customHeaders: { 'x-databricks-org-id': '2' } }], - ['user agent', {}, { userAgentEntry: 'other-app' }], - ['custom header', {}, { customHeaders: { 'X-Custom': 'other' } }], - ['server trust', { checkServerCertificate: false }, {}], - ['custom CA', { customCaCert: 'ca-other' }, {}], - ['client certificate', { clientCert: 'cert-other', clientKey: 'key-other' }, {}], - ['proxy', { proxy: { protocol: 'http', host: 'proxy.test', port: 8080 } }, {}], - ['protocol', { https: false }, {}], - ['port', { port: 8443 }, {}], - ] as Array<[string, Partial, any]>) { - it(`partitions active and idle responses by ${name}`, async () => { - const first = makeCache(); - sinon.stub(first as any, 'fetchWithRetry').resolves(response()); - const original = first.getOrCreateContext(host); - await first.getBoolean(host, 'flag'); - const other = makeCache(overrides, config); - expect(other.getOrCreateContext(host)).to.not.equal(original); - other.releaseContext(host); - first.releaseContext(host); - const next = makeCache(overrides, config); - const fetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response('false')); - next.getOrCreateContext(host); - expect(await next.getBoolean(host, 'flag')).to.equal(false); - expect(fetch.callCount).to.equal(1); - next.releaseContext(host); - }); - } - - it('includes the host even when the workspace is the same', () => { - const cache = makeCache({}, { customHeaders: { 'x-databricks-org-id': '1' } }); - expect(cache.getOrCreateContext(host)).to.not.equal(cache.getOrCreateContext('other.databricks.com')); - cache.releaseContext(host); - cache.releaseContext('other.databricks.com'); + it('partitions active and idle responses by workspace or normalized host, as on main', async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); + const caches = [makeCache(), makeCache(), makeCache({ customHeaders: { 'x-databricks-org-id': '1' } })]; + const hosts = [host, 'other.databricks.com', host]; + const contexts = caches.map((cache, index) => cache.getOrCreateContext(hosts[index])); + expect(new Set(contexts).size).to.equal(3); + await Promise.all(caches.map((cache, index) => cache.getBoolean(hosts[index], 'flag'))); + expect(fetch.callCount).to.equal(3); + caches.forEach((cache, index) => cache.releaseContext(hosts[index])); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(3); }); - it('does not alias an absent token with a literal anonymous token', () => { - const anonymous = makeCache({ token: undefined }); - const token = makeCache({ token: 'anonymous' }); - expect(anonymous.getOrCreateContext(host)).to.not.equal(token.getOrCreateContext(host)); - anonymous.releaseContext(host); - token.releaseContext(host); + it('shares active and idle workspace values across hosts, providers and non-routing headers', async () => { + const firstAuth = { authenticate: sinon.stub().resolves({ Authorization: 'Bearer first-secret' }) }; + const nextAuth = { authenticate: sinon.stub().resolves({ Authorization: 'Bearer next-secret' }) }; + const first = makeCache({ customHeaders: { 'x-databricks-org-id': '1', 'X-Custom': 'first' } }, firstAuth); + const next = makeCache( + { customHeaders: { 'X-Databricks-Org-Id': '1', 'X-Custom': 'next' }, userAgentEntry: 'other-app' }, + nextAuth, + ); + const fetch = sinon.stub(first as any, 'fetchWithRetry').resolves(response()); + const nextFetch = sinon.stub(next as any, 'fetchWithRetry').resolves(response('false')); + const original = first.getOrCreateContext(host); + expect(next.getOrCreateContext('other.databricks.com')).to.equal(original); + expect( + await Promise.all([first.getBoolean(host, 'flag'), next.getBoolean('other.databricks.com', 'flag')]), + ).to.deep.equal([true, true]); + first.releaseContext(host); + next.releaseContext('other.databricks.com'); + clock.tick(2000); + expect(next.getOrCreateContext('third.databricks.com')).to.equal(original); + expect(await next.getBoolean('third.databricks.com', 'flag')).to.equal(true); + expect(fetch.callCount).to.equal(1); + expect(nextFetch.callCount).to.equal(0); + clock.tick(58001); + expect(await next.getBoolean('third.databricks.com', 'flag')).to.equal(false); + expect(nextAuth.authenticate.callCount).to.equal(1); + expect(nextFetch.callCount).to.equal(1); + next.releaseContext('third.databricks.com'); + expect(JSON.stringify([...(FeatureFlagCache as any).idleContexts])).to.not.contain('secret'); }); - it('canonicalizes host and custom header casing and order', () => { - const first = makeCache({}, { customHeaders: { 'X-Route': 'a', 'X-Databricks-Org-Id': '1' } }); - const next = makeCache({}, { customHeaders: { 'x-databricks-org-id': '1', 'x-route': 'a' } }); - expect(first.getOrCreateContext(host)).to.equal(next.getOrCreateContext(`https://${host.toUpperCase()}:443/`)); + it('canonicalizes fallback hosts across an idle reconnect', async () => { + const first = makeCache(); + sinon.stub(first as any, 'fetchWithRetry').resolves(response()); + const original = first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); first.releaseContext(host); + const next = makeCache(); + expect(next.getOrCreateContext(`https://${host.toUpperCase()}:443/`)).to.equal(original); next.releaseContext(host); }); - it('never retains a dynamic or deprecated authentication provider across close', async () => { - const auth = { authenticate: sinon.stub().resolves({ Authorization: 'Bearer changing-token' }) }; - for (let iteration = 0; iteration < 2; iteration += 1) { - const cache = makeCache({}, {}, auth); - const fetch = sinon.stub(cache as any, 'fetchWithRetry').resolves(response(iteration === 0 ? 'true' : 'false')); - cache.getOrCreateContext(host); - expect(await cache.getBoolean(host, 'flag')).to.equal(iteration === 0); + for (const authType of [ + 'access-token', + 'databricks-oauth', + 'token-provider', + 'external-token', + 'static-token', + 'custom', + ]) { + it(`deduplicates 50 distinct ${authType} clients before and after idle`, async () => { + const oauth = sinon.stub(DatabricksOAuth.prototype, 'authenticate').resolves({ Authorization: 'Bearer oauth' }); + sinon.stub(ThriftBackend.prototype, 'connect').resolves(); + sinon.stub(ThriftBackend.prototype, 'close').resolves(); + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); + const burst = async () => { + const clients = Array.from({ length: 50 }, () => new DBSQLClient({ logger: { log: () => {} } })); + await Promise.all( + clients.map((client, index) => { + const common = { host, path: options.path, telemetryEnabled: false }; + const token = `distinct-secret-${index}`; + let connection: ConnectionOptions; + switch (authType) { + case 'databricks-oauth': + connection = { ...common, authType, oauthClientId: 'test-id', oauthClientSecret: 'test-secret' }; + break; + case 'token-provider': + connection = { + ...common, + authType, + tokenProvider: new StaticTokenProvider(token), + enableTokenFederation: false, + }; + break; + case 'external-token': + connection = { ...common, authType, getToken: async () => token, enableTokenFederation: false }; + break; + case 'static-token': + connection = { ...common, authType, staticToken: token, enableTokenFederation: false }; + break; + case 'custom': + connection = { + ...common, + authType, + provider: { authenticate: async () => ({ Authorization: `Bearer ${token}` }) }, + }; + break; + default: + connection = { ...common, token }; + } + return client.connect(connection); + }), + ); + expect( + await Promise.all(clients.map((client) => (client as any).getFeatureFlagCache().getBoolean(host, 'flag'))), + ).to.deep.equal(Array(50).fill(true)); + expect((FeatureFlagCache as any).sharedContexts.size).to.equal(1); + await Promise.all(clients.map((client) => client.close())); + }; + await burst(); expect(fetch.callCount).to.equal(1); - cache.releaseContext(host); - } - expect((FeatureFlagCache as any).idleContexts.size).to.equal(0); - expect(auth.authenticate.callCount).to.equal(2); - }); + clock.tick(2000); + await burst(); + expect(fetch.callCount).to.equal(1); + expect(oauth.callCount).to.equal(authType === 'databricks-oauth' ? 1 : 0); + expect((FeatureFlagCache as any).sharedContexts.size).to.equal(0); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(1); + }); + } for (const [name, body, status] of [ ['missing flags', {}, 200], @@ -282,10 +338,15 @@ describe('FeatureFlagCache idle retention', () => { expect((FeatureFlagCache as any).idleContexts.size).to.equal(0); }); - it('bounds retained contexts after 1000 distinct identities without retaining credentials', async () => { + it('bounds retained contexts after 1000 distinct workspaces without retaining credentials', async () => { sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); for (let iteration = 0; iteration < 1000; iteration += 1) { - const cache = makeCache({ token: `secret-identity-${iteration}` }); + const cache = makeCache( + { customHeaders: { 'x-databricks-org-id': String(iteration) } }, + { + authenticate: async () => ({ Authorization: 'Bearer secret-identity' }), + }, + ); cache.getOrCreateContext(host); expect(await cache.getBoolean(host, 'flag')).to.equal(true); cache.releaseContext(host); diff --git a/tests/unit/telemetry/FeatureFlagCache.test.ts b/tests/unit/telemetry/FeatureFlagCache.test.ts index 65ac21ac..033e710d 100644 --- a/tests/unit/telemetry/FeatureFlagCache.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.test.ts @@ -409,11 +409,11 @@ describe('FeatureFlagCache', () => { expect(fetchStub.calledOnce).to.be.true; }); - it('shares values for the same host and workspace, refreshes with the current caller, and defaults on failure', async () => { + it('shares values per workspace, refreshes with the current caller, and defaults on failure', async () => { const first = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'x-databricks-org-id': '1' } })); const current = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'X-Databricks-Org-Id': '1' } })); const other = new FeatureFlagCache(new ClientContextStub({ customHeaders: { 'x-databricks-org-id': '2' } })); - expect(first.getOrCreateContext('host-a')).to.equal(current.getOrCreateContext('host-a')); + expect(first.getOrCreateContext('host-a')).to.equal(current.getOrCreateContext('host-b')); expect(other.getOrCreateContext('host-a')).to.not.equal(first.getOrCreateContext('host-a')); const response = (value: string) => new Response( @@ -424,18 +424,18 @@ describe('FeatureFlagCache', () => { ); const firstFetch = sinon.stub(first as any, 'fetchWithRetry').resolves(response('true')); const currentFetch = sinon.stub(current as any, 'fetchWithRetry').resolves(response('FALSE')); - expect(await Promise.all([first.getBoolean('host-a', 'flag'), current.getBoolean('host-a', 'flag')])).to.deep.equal( + expect(await Promise.all([first.getBoolean('host-a', 'flag'), current.getBoolean('host-b', 'flag')])).to.deep.equal( [true, true], ); expect(currentFetch.called).to.be.false; clock.tick(61000); - expect(await current.getBoolean('host-a', 'flag', true)).to.be.false; + expect(await current.getBoolean('host-b', 'flag', true)).to.be.false; expect(firstFetch.calledOnce).to.be.true; expect(currentFetch.calledOnce).to.be.true; clock.tick(61000); currentFetch.rejects(new Error('offline')); - expect(await current.getBoolean('host-a', 'flag', true)).to.be.true; - expect(await current.getBoolean('host-a', 'flag')).to.be.false; + expect(await current.getBoolean('host-b', 'flag', true)).to.be.true; + expect(await current.getBoolean('host-b', 'flag')).to.be.false; expect(currentFetch.calledTwice).to.be.true; }); }); From 6fe619535c326bfdafa3a70b7f2f776917003584 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 09:36:09 +0000 Subject: [PATCH 4/5] Reset feature-flag refresh TTL and pin idle cache lifecycle Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/FeatureFlagCache.ts | 7 +- tests/e2e/feature_flag_cache.test.ts | 26 ++-- .../FeatureFlagCache.retention.test.ts | 144 +++++++++++++++++- 4 files changed, 161 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85b96d68..444e0feb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Reuse successful Thrift feature-flag responses across connections to the same workspace (or host), using the existing sharing key for every auth type. Idle responses are bounded and expire at the original server TTL (currently up to one hour), as for a long-lived client; they do not retain connections or timers. +Reuse successful Thrift feature-flag responses across connections to the same workspace (or host) for every auth type, retaining at most 100 idle responses until their original server TTL (currently up to one hour) without keeping connections or timers. For sequential reconnects, `CONNECTION_OPEN` telemetry latency now includes a TLS handshake (~70 ms), so this metric can rise while total connect+openSession latency falls. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index 66eebfb6..b39c07e5 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -123,7 +123,7 @@ export default class FeatureFlagCache { const isExpired = this.isExpired(ctx); - if (isExpired) { + if (ctx.fetchPromise || isExpired) { if (!ctx.fetchPromise) { ctx.fetchPromise = this.fetchFeatureFlags(host) .then((flags) => { @@ -200,7 +200,10 @@ export default class FeatureFlagCache { private async fetchFeatureFlags(host: string): Promise> { const logger = this.context.getLogger(); const ctx = this.contexts.get(this.cacheKey(host)); - if (ctx) ctx.cacheable = false; + if (ctx) { + ctx.cacheable = false; + ctx.cacheDuration = this.CACHE_DURATION_MS; + } try { const endpoint = buildTelemetryUrl( diff --git a/tests/e2e/feature_flag_cache.test.ts b/tests/e2e/feature_flag_cache.test.ts index 90892b4c..ebc94e81 100644 --- a/tests/e2e/feature_flag_cache.test.ts +++ b/tests/e2e/feature_flag_cache.test.ts @@ -1,26 +1,28 @@ import { expect } from 'chai'; -import { channel } from 'diagnostics_channel'; +import sinon from 'sinon'; import { DBSQLClient } from '../../lib'; import config from './utils/config'; import FeatureFlagCache from '../../lib/FeatureFlagCache'; describe('Thrift feature flags across idle reconnects', () => { + let sharedContexts: sinon.SinonStub; + let idleContexts: sinon.SinonStub; + beforeEach(() => { - (FeatureFlagCache as any).idleContexts.clear(); + sharedContexts = sinon.stub(FeatureFlagCache as any, 'sharedContexts').value(new Map()); + idleContexts = sinon.stub(FeatureFlagCache as any, 'idleContexts').value(new Map()); + }); + + afterEach(() => { + idleContexts.restore(); + sharedContexts.restore(); }); for (const reuseClient of [false, true]) { it(`reuses flags and preserves results when ${ reuseClient ? 'reconnecting a client' : 'creating another client' }`, async () => { - let requests = 0; - const onRequest = ({ request }: any) => { - if (request.method === 'GET' && request.path.includes('/connector-service/feature-flags/NODEJS/')) { - requests += 1; - } - }; - const requestsChannel = channel('http.client.request.start'); - requestsChannel.subscribe(onRequest); + const fetch = sinon.spy(FeatureFlagCache.prototype as any, 'fetchFeatureFlags'); let client = new DBSQLClient(); let enabled: boolean | undefined; try { @@ -52,9 +54,9 @@ describe('Thrift feature flags across idle reconnects', () => { }; await cycle(0); await cycle(1); - expect(requests).to.equal(1); + expect(fetch.callCount).to.equal(1); } finally { - requestsChannel.unsubscribe(onRequest); + fetch.restore(); } }); } diff --git a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts index 0216071a..1dde862f 100644 --- a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; import { Response } from 'node-fetch'; -import FeatureFlagCache from '../../../lib/FeatureFlagCache'; +import FeatureFlagCache, { FeatureFlagContext } from '../../../lib/FeatureFlagCache'; import ClientContextStub from '../.stubs/ClientContextStub'; import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; import DBSQLClient from '../../../lib/DBSQLClient'; @@ -35,6 +35,17 @@ describe('FeatureFlagCache idle retention', () => { return new Response(JSON.stringify({ flags: [{ name: 'flag', value }], ttl_seconds: ttl })); } + function expectRetainedContext(context: FeatureFlagContext, value = 'true') { + expect(context).to.have.all.keys('flags', 'fetchPromise', 'lastFetched', 'cacheable', 'refCount', 'cacheDuration'); + expect(context.flags).to.be.instanceOf(Map); + expect([...context.flags!]).to.deep.equal([['flag', value]]); + expect(context.fetchPromise).to.be.undefined; + expect(context.lastFetched).to.be.a('number'); + expect(context.cacheable).to.equal(true); + expect(context.refCount).to.equal(0); + expect(context.cacheDuration).to.be.a('number').and.greaterThan(0); + } + it('reuses a successful response after the last consumer closes', async () => { const first = makeCache(); const fetch = sinon.stub(first as any, 'fetchWithRetry').resolves(response()); @@ -79,6 +90,133 @@ describe('FeatureFlagCache idle retention', () => { cache.releaseContext(host); }); + it('expires from fetch time even when release occurs 50 seconds later', async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry'); + fetch.onFirstCall().resolves(response()); + fetch.onSecondCall().resolves(response('false')); + const first = makeCache(); + first.getOrCreateContext(host); + expect(await first.getBoolean(host, 'flag')).to.equal(true); + clock.tick(50000); + first.releaseContext(host); + clock.tick(9999); + const beforeExpiry = makeCache(); + beforeExpiry.getOrCreateContext(host); + expect(await beforeExpiry.getBoolean(host, 'flag')).to.equal(true); + expect(fetch.callCount).to.equal(1); + beforeExpiry.releaseContext(host); + clock.tick(2); + const afterExpiry = makeCache(); + afterExpiry.getOrCreateContext(host); + expect(await afterExpiry.getBoolean(host, 'flag')).to.equal(false); + expect(fetch.callCount).to.equal(2); + afterExpiry.releaseContext(host); + }); + + for (const ttl of [undefined, 0, '3600']) { + it(`restores the default TTL when a reused context refreshes with ttl_seconds=${ttl}`, async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry'); + fetch.onFirstCall().resolves(response('true', 3600)); + fetch + .onSecondCall() + .resolves(new Response(JSON.stringify({ flags: [{ name: 'flag', value: 'false' }], ttl_seconds: ttl }))); + fetch.onThirdCall().resolves(response()); + const first = makeCache(); + first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); + first.releaseContext(host); + clock.tick(1000); + const reused = makeCache(); + const context = reused.getOrCreateContext(host); + clock.tick(3600000); + expect(await reused.getBoolean(host, 'flag')).to.equal(false); + expect(context.cacheDuration).to.equal(15 * 60 * 1000); + reused.releaseContext(host); + clock.tick(15 * 60 * 1000 + 1); + const refreshed = makeCache(); + refreshed.getOrCreateContext(host); + expect(await refreshed.getBoolean(host, 'flag')).to.equal(true); + expect(fetch.callCount).to.equal(3); + refreshed.releaseContext(host); + }); + } + + it('waits for a shared refresh even when its TTL resets to a longer default', async () => { + let complete!: (response: Response) => void; + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry'); + fetch.onFirstCall().resolves(response()); + fetch.onSecondCall().returns( + new Promise((resolve) => { + complete = resolve; + }), + ); + const cache = makeCache(); + cache.getOrCreateContext(host); + expect(await cache.getBoolean(host, 'flag')).to.equal(true); + clock.tick(61000); + const firstRead = cache.getBoolean(host, 'flag'); + await clock.tickAsync(0); + let settled = false; + const secondRead = cache.getBoolean(host, 'flag').then((value) => { + settled = true; + return value; + }); + await clock.tickAsync(0); + expect(settled).to.equal(false); + complete(response('false')); + expect(await Promise.all([firstRead, secondRead])).to.deep.equal([false, false]); + expect(fetch.callCount).to.equal(2); + cache.releaseContext(host); + }); + + it('does not retain a reused context after a failed refresh', async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry'); + fetch.onFirstCall().resolves(response('true', 3600)); + fetch.onSecondCall().resolves(new Response('{}', { status: 500 })); + fetch.onThirdCall().resolves(response('false')); + const first = makeCache(); + first.getOrCreateContext(host); + await first.getBoolean(host, 'flag'); + first.releaseContext(host); + const reused = makeCache(); + const context = reused.getOrCreateContext(host); + clock.tick(3600001); + expect(await reused.getBoolean(host, 'flag')).to.equal(false); + expect(context.cacheDuration).to.equal(15 * 60 * 1000); + reused.releaseContext(host); + expect((FeatureFlagCache as any).idleContexts.size).to.equal(0); + const next = makeCache(); + next.getOrCreateContext(host); + expect(await next.getBoolean(host, 'flag')).to.equal(false); + expect(fetch.callCount).to.equal(3); + next.releaseContext(host); + }); + + it('evicts the oldest idle key and moves a re-released key to the end', async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); + const visit = async (workspace: number) => { + const cache = makeCache({ customHeaders: { 'x-databricks-org-id': String(workspace) } }); + cache.getOrCreateContext(host); + expect(await cache.getBoolean(host, 'flag')).to.equal(true); + cache.releaseContext(host); + }; + for (let workspace = 0; workspace < 100; workspace += 1) await visit(workspace); + expect(fetch.callCount).to.equal(100); + await visit(0); + expect(fetch.callCount).to.equal(100); + await visit(100); + const idle = (FeatureFlagCache as any).idleContexts as Map; + expect(idle.size).to.equal(100); + expect(idle.has('workspace:0')).to.equal(true); + expect(idle.has('workspace:1')).to.equal(false); + expect(idle.has('workspace:100')).to.equal(true); + expect(idle.keys().next().value).to.equal('workspace:2'); + await visit(0); + expect(fetch.callCount).to.equal(101); + await visit(1); + expect(fetch.callCount).to.equal(102); + }); + it('preserves every typed reader and null/missing defaults after close', async () => { const cases: Array<[string, string, any]> = [ ['getBoolean', 'True', true], @@ -187,7 +325,7 @@ describe('FeatureFlagCache idle retention', () => { expect(nextAuth.authenticate.callCount).to.equal(1); expect(nextFetch.callCount).to.equal(1); next.releaseContext('third.databricks.com'); - expect(JSON.stringify([...(FeatureFlagCache as any).idleContexts])).to.not.contain('secret'); + expectRetainedContext(original, 'false'); }); it('canonicalizes fallback hosts across an idle reconnect', async () => { @@ -353,7 +491,7 @@ describe('FeatureFlagCache idle retention', () => { } expect((FeatureFlagCache as any).sharedContexts.size).to.equal(0); expect((FeatureFlagCache as any).idleContexts.size).to.equal(100); - expect(JSON.stringify([...(FeatureFlagCache as any).idleContexts])).to.not.contain('secret-identity'); + for (const context of (FeatureFlagCache as any).idleContexts.values()) expectRetainedContext(context); clock.tick(61000); const next = makeCache(); next.getOrCreateContext(host); From c712c078a300097b71f2687d18ce9d7593f5dcc4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 09:48:52 +0000 Subject: [PATCH 5/5] Preserve shorter feature-flag retry windows after failed refreshes Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/FeatureFlagCache.ts | 5 ++++ .../FeatureFlagCache.retention.test.ts | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 444e0feb..60a40067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Reuse successful Thrift feature-flag responses across connections to the same workspace (or host) for every auth type, retaining at most 100 idle responses until their original server TTL (currently up to one hour) without keeping connections or timers. For sequential reconnects, `CONNECTION_OPEN` telemetry latency now includes a TLS handshake (~70 ms), so this metric can rise while total connect+openSession latency falls. +- Reuse successful Thrift feature-flag responses across connections to the same workspace (or host) for every auth type, retaining at most 100 idle responses until their original server TTL (currently up to one hour) without keeping connections or timers. For sequential reconnects, `CONNECTION_OPEN` telemetry latency now includes a TLS handshake (~70 ms), so this metric can rise while total connect+openSession latency falls. (#630) - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/FeatureFlagCache.ts b/lib/FeatureFlagCache.ts index b39c07e5..9c30bb9b 100644 --- a/lib/FeatureFlagCache.ts +++ b/lib/FeatureFlagCache.ts @@ -200,6 +200,7 @@ export default class FeatureFlagCache { private async fetchFeatureFlags(host: string): Promise> { const logger = this.context.getLogger(); const ctx = this.contexts.get(this.cacheKey(host)); + const previousCacheDuration = ctx?.cacheDuration ?? this.CACHE_DURATION_MS; if (ctx) { ctx.cacheable = false; ctx.cacheDuration = this.CACHE_DURATION_MS; @@ -262,6 +263,10 @@ export default class FeatureFlagCache { // Preserve the existing policy: use consumer defaults until the next TTL. // Retrying on every read would add request latency during an outage. return new Map(); + } finally { + if (ctx && !ctx.cacheable) { + ctx.cacheDuration = Math.min(previousCacheDuration, this.CACHE_DURATION_MS); + } } } diff --git a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts index 1dde862f..5e0ead21 100644 --- a/tests/unit/telemetry/FeatureFlagCache.retention.test.ts +++ b/tests/unit/telemetry/FeatureFlagCache.retention.test.ts @@ -192,6 +192,29 @@ describe('FeatureFlagCache idle retention', () => { next.releaseContext(host); }); + for (const [name, body, status] of [ + ['HTTP failure', '{}', 503], + ['missing flags', '{}', 200], + ['invalid JSON', '{', 200], + ] as Array<[string, string, number]>) { + it(`preserves a shorter retry cadence after ${name}`, async () => { + const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry'); + fetch.onFirstCall().resolves(response()); + fetch.onSecondCall().resolves(new Response(body, { status })); + fetch.onThirdCall().resolves(response()); + const cache = makeCache(); + const context = cache.getOrCreateContext(host); + expect(await cache.getBoolean(host, 'flag')).to.equal(true); + clock.tick(61000); + expect(await cache.getBoolean(host, 'flag')).to.equal(false); + expect(context.cacheDuration).to.equal(60000); + clock.tick(60001); + expect(await cache.getBoolean(host, 'flag')).to.equal(true); + expect(fetch.callCount).to.equal(3); + cache.releaseContext(host); + }); + } + it('evicts the oldest idle key and moves a re-released key to the end', async () => { const fetch = sinon.stub(FeatureFlagCache.prototype as any, 'fetchWithRetry').callsFake(async () => response()); const visit = async (workspace: number) => {