From 3bf0c20e45c9ddb5924aa4f237ad48d8b199a254 Mon Sep 17 00:00:00 2001 From: Karl Power Date: Mon, 16 Nov 2026 11:59:38 +0100 Subject: [PATCH] feat: add per-source row key override for the side-panel row lookup Add an optional rowKeyExpression to log and trace sources. When set, the row lookup selects and filters on only those expressions (plus their column references) and skips the table metadata, columns, and MergeTree settings queries. This covers sources whose table metadata hides the keys, e.g. Distributed over Merge over SharedMergeTree, where the lookup otherwise falls back to a full-row WHERE including Body. Also exposed in MCP and the external API docs. Unset sources behave as before. --- .changeset/source-row-key-expression.md | 9 +++ packages/api/openapi.json | 12 +++ .../api/src/mcp/__tests__/sources.int.test.ts | 8 ++ packages/api/src/mcp/tools/sources/schemas.ts | 9 +++ packages/api/src/models/source.ts | 2 + .../src/routers/external-api/v2/sources.ts | 10 +++ packages/app/src/components/DBRowTable.tsx | 49 +++++++++-- .../Sources/SourceForm/LogTableModelForm.tsx | 17 ++++ .../SourceForm/TraceTableModelForm.tsx | 17 ++++ .../Sources/SourceForm/constants.ts | 3 + .../components/__tests__/DBRowTable.test.tsx | 81 +++++++++++++++++++ packages/common-utils/src/types.ts | 7 ++ 12 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 .changeset/source-row-key-expression.md diff --git a/.changeset/source-row-key-expression.md b/.changeset/source-row-key-expression.md new file mode 100644 index 0000000000..66c39403d8 --- /dev/null +++ b/.changeset/source-row-key-expression.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +feat: add an optional "Row key expression" to log and trace sources. When set, +the side panel's row lookup selects and filters on only those columns instead of +detecting keys from table metadata. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3faea..8e883b9827 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5192,6 +5192,12 @@ "nullable": true, "example": "Timestamp, Body, ServiceName" }, + "rowKeyExpression": { + "type": "string", + "description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.", + "nullable": true, + "example": "toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ @@ -5446,6 +5452,12 @@ "nullable": true, "example": "Timestamp, Body, ServiceName" }, + "rowKeyExpression": { + "type": "string", + "description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.", + "nullable": true, + "example": "toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset" + }, "useTextIndexForImplicitColumn": { "type": "string", "enum": [ diff --git a/packages/api/src/mcp/__tests__/sources.int.test.ts b/packages/api/src/mcp/__tests__/sources.int.test.ts index 59367443b3..3f710f1694 100644 --- a/packages/api/src/mcp/__tests__/sources.int.test.ts +++ b/packages/api/src/mcp/__tests__/sources.int.test.ts @@ -1451,6 +1451,8 @@ describe('MCP Source Tools', () => { // unwritable through save_source). useTextIndexForImplicitColumn: 'enabled', knownColumnsListExpression: 'TraceId, SpanId', + rowKeyExpression: + 'ServiceName, Timestamp, _block_number, _block_offset', sampleRateExpression: 'SampleRate', highlightedTraceAttributeExpressions: [ { sqlExpression: "SpanAttributes['http.method']", alias: 'method' }, @@ -1480,6 +1482,9 @@ describe('MCP Source Tools', () => { // Advanced fields surface too. expect(config.useTextIndexForImplicitColumn).toBe('enabled'); expect(config.knownColumnsListExpression).toBe('TraceId, SpanId'); + expect(config.rowKeyExpression).toBe( + 'ServiceName, Timestamp, _block_number, _block_offset', + ); expect(config.sampleRateExpression).toBe('SampleRate'); expect(config.highlightedTraceAttributeExpressions).toEqual([ { sqlExpression: "SpanAttributes['http.method']", alias: 'method' }, @@ -1514,6 +1519,9 @@ describe('MCP Source Tools', () => { ); expect(stored?.get('useTextIndexForImplicitColumn')).toBe('enabled'); expect(stored?.get('sampleRateExpression')).toBe('SampleRate'); + expect(stored?.get('rowKeyExpression')).toBe( + 'ServiceName, Timestamp, _block_number, _block_offset', + ); // 4. describe the clone: advanced config round-trips identically. const describedClone = await callTool( diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 03f293efa4..c095cb6a15 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -254,6 +254,15 @@ const mcpSaveSourceSchema = z.object({ 'targets, used instead of SELECT * when fetching full rows. Column ' + 'names only, no expressions or aliases. Omit to select all columns.', ), + rowKeyExpression: z + .string() + .optional() + .describe( + 'Log and trace sources. Comma-separated expressions that together ' + + 'identify one row, e.g. the partition key, primary key, _block_number ' + + 'and _block_offset. When set, the row side panel looks rows up by ' + + 'these only. Omit to detect keys from table metadata.', + ), orderByExpression: z .string() .optional() diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 6b162640d7..42d8e44b7a 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -170,6 +170,7 @@ export const LogSource = Source.discriminator( spanIdExpression: String, implicitColumnExpression: String, knownColumnsListExpression: String, + rowKeyExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), @@ -222,6 +223,7 @@ export const TraceSource = Source.discriminator( spanLinksValueExpression: String, implicitColumnExpression: String, knownColumnsListExpression: String, + rowKeyExpression: String, useTextIndexForImplicitColumn: { type: String, enum: Object.values(UseTextIndex), diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 561625bc65..8da4fd0b4e 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -406,6 +406,11 @@ function formatExternalSource(source: SourceDocument) { * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. * nullable: true * example: Timestamp, Body, ServiceName + * rowKeyExpression: + * type: string + * description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata. + * nullable: true + * example: toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] @@ -607,6 +612,11 @@ function formatExternalSource(source: SourceDocument) { * description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns. * nullable: true * example: Timestamp, Body, ServiceName + * rowKeyExpression: + * type: string + * description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata. + * nullable: true + * example: toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset * useTextIndexForImplicitColumn: * type: string * enum: [auto, enabled, disabled] diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 2c90cbf204..c20b1586ec 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -31,6 +31,8 @@ import { import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, + isLogSource, + isTraceSource, SelectList, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -1576,15 +1578,27 @@ function getSelectLength(select: SelectList): number { } } +function getRowKeyExpression(source: TSource | undefined): string | undefined { + return source && (isLogSource(source) || isTraceSource(source)) + ? source.rowKeyExpression?.trim() || undefined + : undefined; +} + export function useConfigWithAdditionalSelect( config: BuilderChartConfigWithDateRange, sourceId?: string, + rowKeyExpression?: string, ) { - const { data: tableMetadata } = useTableMetadata({ - databaseName: config.from.databaseName, - tableName: config.from.tableName, - connectionId: config.connection, - }); + const hasRowKeyOverride = !!sourceId && !!rowKeyExpression; + + const { data: tableMetadata } = useTableMetadata( + { + databaseName: config.from.databaseName, + tableName: config.from.tableName, + connectionId: config.connection, + }, + hasRowKeyOverride ? { enabled: false } : undefined, + ); // Only check for row-ID columns for row-level queries (sourceId present). // Skip for aggregate queries (e.g. patterns) where extra keys are irrelevant. @@ -1594,12 +1608,13 @@ export function useConfigWithAdditionalSelect( tableName: config.from.tableName, connectionId: config.connection, }, - { enabled: !!sourceId }, + { enabled: !!sourceId && !hasRowKeyOverride }, ); const engineFull = tableMetadata?.engine_full ?? ''; const needsMergeTreeSettings = !!sourceId && + !hasRowKeyOverride && !!tableMetadata?.engine?.includes('MergeTree') && omitsBlockColumnSetting(engineFull); @@ -1612,6 +1627,19 @@ export function useConfigWithAdditionalSelect( const partitionKey = tableMetadata?.partition_key; return useMemo(() => { + if (sourceId && rowKeyExpression) { + const { select, additionalKeysLength } = appendSelectWithAdditionalKeys( + config.select, + rowKeyExpression, + '', + ); + const rowKeyColumns = new Set([ + ...splitAndTrimWithBracket(rowKeyExpression), + ...extractColumnReferencesFromKey(rowKeyExpression), + ]); + return { ...config, select, additionalKeysLength, rowKeyColumns }; + } + // Wait for settings to avoid querying twice. if ( primaryKey == null || @@ -1674,6 +1702,7 @@ export function useConfigWithAdditionalSelect( columns, mergeTreeSettings, sourceId, + rowKeyExpression, ]); } @@ -1821,7 +1850,12 @@ function DBSqlRowTableComponent({ return base; }, [me, config, orderByArray]); - const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj, sourceId); + const { data: source } = useSource({ id: sourceId }); + const mergedConfig = useConfigWithAdditionalSelect( + mergedConfigObj, + sourceId, + getRowKeyExpression(source), + ); const { data, fetchNextPage, hasNextPage, isFetching, isError, error } = useOffsetPaginatedQuery(mergedConfig ?? config, { @@ -1943,7 +1977,6 @@ function DBSqlRowTableComponent({ } }, [data?.meta, onResolvedColumnsChange]); - const { data: source } = useSource({ id: sourceId }); const denoised = useDenoisedRows({ config, source, diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index b936c352f3..58c2826ba7 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -14,6 +14,7 @@ import { DEFAULT_DATABASE, EVENT_NAME_EXPRESSION_HELP_TEXT, KNOWN_COLUMNS_EXPRESSION_HELP_TEXT, + ROW_KEY_EXPRESSION_HELP_TEXT, SERVICE_VERSION_EXPRESSION_HELP_TEXT, SEVERITY_NUMBER_EXPRESSION_HELP_TEXT, } from './constants'; @@ -285,6 +286,22 @@ export function LogTableModelForm(props: TableModelProps) { disableKeywordAutocomplete /> + + + + + + = { export const KNOWN_COLUMNS_EXPRESSION_HELP_TEXT = 'For Distributed table sources whose target tables have non-matching column sets. Provide a list of columns supported across all target tables; it is used instead of SELECT * when fetching full row data (e.g. the row side panel). Leave blank to select all columns. This should be a comma-separated list of column names - do not include non-column expressions or aliases.'; +export const ROW_KEY_EXPRESSION_HELP_TEXT = + 'Comma-separated columns that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only.'; + export const SERVICE_VERSION_EXPRESSION_HELP_TEXT = "Identifies the running release of a service. Defaults to the OpenTelemetry service.version resource attribute. Point it elsewhere if your release identifier lives in another attribute - under GitOps it is often the container image tag. If services in this table use different attributes, fall back across them: coalesce(nullIf(ResourceAttributes['service.version'], ''), nullIf(ResourceAttributes['container.image.tag'], ''))."; diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index b6232dbbc6..8b8290942d 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -1198,4 +1198,85 @@ describe('useConfigWithAdditionalSelect', () => { 'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_offset', ); }); + + describe('with a row key expression', () => { + const metadataLoading = () => { + mockUseTableMetadata.mockReturnValue({ data: undefined }); + mockUseColumns.mockReturnValue({ data: undefined }); + mockUseMergeTreeSettings.mockReturnValue({ + data: undefined, + isLoading: true, + }); + }; + + it('restricts the row lookup to the listed columns', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'ServiceName, Timestamp, _block_number, _block_offset', + ), + ); + + expect(result.current?.select).toBe( + 'Timestamp,ServiceName,Body,_block_number,_block_offset', + ); + expect(result.current?.additionalKeysLength).toBe(2); + expect(result.current?.rowKeyColumns).toEqual( + new Set(['ServiceName', 'Timestamp', '_block_number', '_block_offset']), + ); + expect(result.current?.rowKeyColumns?.has('Body')).toBe(false); + }); + + it('returns a config without waiting for table metadata or settings', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'Timestamp, _block_number, _block_offset', + ), + ); + + expect(mockUseTableMetadata).toHaveBeenCalledWith(expect.anything(), { + enabled: false, + }); + expect(mockUseColumns).toHaveBeenCalledWith(expect.anything(), { + enabled: false, + }); + expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', { + enabled: false, + }); + expect(result.current).toBeDefined(); + }); + + it('does not split inside function calls', () => { + metadataLoading(); + + const { result } = renderHook(() => + useConfigWithAdditionalSelect( + config, + 'source-id', + 'toYYYYMM(event_date), service, _block_number, _block_offset', + ), + ); + + expect(result.current?.select).toBe( + 'Timestamp,ServiceName,Body,event_date,service,_block_number,_block_offset,toYYYYMM(event_date)', + ); + expect(result.current?.additionalKeysLength).toBe(5); + expect(result.current?.rowKeyColumns).toEqual( + new Set([ + 'toYYYYMM(event_date)', + 'service', + '_block_number', + '_block_offset', + 'event_date', + ]), + ); + }); + }); }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dba9c7854e..d21d64df04 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2737,6 +2737,11 @@ export const LogSourceSchema = BaseSourceSchema.extend({ spanIdExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), knownColumnsListExpression: z.string().optional(), + /** + * Comma-separated expressions that together identify one row. When set, the + * row side panel looks rows up by these instead of table metadata. + */ + rowKeyExpression: z.string().optional(), /** * @deprecated Application-side SQL predicate AND'd into every query against * the source. Not a security boundary; bypassable by direct table SELECT. @@ -2791,6 +2796,8 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ spanLinksValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), knownColumnsListExpression: z.string().optional(), + /** See `rowKeyExpression` on `LogSourceSchema`. */ + rowKeyExpression: z.string().optional(), displayedTimestampValueExpression: z.string().optional(), highlightedTraceAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(),