Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/source-row-key-expression.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@hyperdx/common-utils': minor
'@hyperdx/api': minor
'@hyperdx/app': minor
---

feat: add an optional "Row key expression" to log and trace sources. When set,
the side panel's row lookup selects and filters on only those columns instead of
detecting keys from table metadata.
12 changes: 12 additions & 0 deletions packages/api/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -5192,6 +5192,12 @@
"nullable": true,
"example": "Timestamp, Body, ServiceName"
},
"rowKeyExpression": {
"type": "string",
"description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.",
"nullable": true,
"example": "toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset"
},
"useTextIndexForImplicitColumn": {
"type": "string",
"enum": [
Expand Down Expand Up @@ -5446,6 +5452,12 @@
"nullable": true,
"example": "Timestamp, Body, ServiceName"
},
"rowKeyExpression": {
"type": "string",
"description": "Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.",
"nullable": true,
"example": "toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset"
},
"useTextIndexForImplicitColumn": {
"type": "string",
"enum": [
Expand Down
8 changes: 8 additions & 0 deletions packages/api/src/mcp/__tests__/sources.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1451,6 +1451,8 @@ describe('MCP Source Tools', () => {
// unwritable through save_source).
useTextIndexForImplicitColumn: 'enabled',
knownColumnsListExpression: 'TraceId, SpanId',
rowKeyExpression:
'ServiceName, Timestamp, _block_number, _block_offset',
sampleRateExpression: 'SampleRate',
highlightedTraceAttributeExpressions: [
{ sqlExpression: "SpanAttributes['http.method']", alias: 'method' },
Expand Down Expand Up @@ -1480,6 +1482,9 @@ describe('MCP Source Tools', () => {
// Advanced fields surface too.
expect(config.useTextIndexForImplicitColumn).toBe('enabled');
expect(config.knownColumnsListExpression).toBe('TraceId, SpanId');
expect(config.rowKeyExpression).toBe(
'ServiceName, Timestamp, _block_number, _block_offset',
);
expect(config.sampleRateExpression).toBe('SampleRate');
expect(config.highlightedTraceAttributeExpressions).toEqual([
{ sqlExpression: "SpanAttributes['http.method']", alias: 'method' },
Expand Down Expand Up @@ -1514,6 +1519,9 @@ describe('MCP Source Tools', () => {
);
expect(stored?.get('useTextIndexForImplicitColumn')).toBe('enabled');
expect(stored?.get('sampleRateExpression')).toBe('SampleRate');
expect(stored?.get('rowKeyExpression')).toBe(
'ServiceName, Timestamp, _block_number, _block_offset',
);

// 4. describe the clone: advanced config round-trips identically.
const describedClone = await callTool(
Expand Down
9 changes: 9 additions & 0 deletions packages/api/src/mcp/tools/sources/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,15 @@ const mcpSaveSourceSchema = z.object({
'targets, used instead of SELECT * when fetching full rows. Column ' +
'names only, no expressions or aliases. Omit to select all columns.',
),
rowKeyExpression: z
.string()
.optional()
.describe(
'Log and trace sources. Comma-separated expressions that together ' +
'identify one row, e.g. the partition key, primary key, _block_number ' +
'and _block_offset. When set, the row side panel looks rows up by ' +
'these only. Omit to detect keys from table metadata.',
),
orderByExpression: z
.string()
.optional()
Expand Down
2 changes: 2 additions & 0 deletions packages/api/src/models/source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ export const LogSource = Source.discriminator<ILogSource>(
spanIdExpression: String,
implicitColumnExpression: String,
knownColumnsListExpression: String,
rowKeyExpression: String,
useTextIndexForImplicitColumn: {
type: String,
enum: Object.values(UseTextIndex),
Expand Down Expand Up @@ -222,6 +223,7 @@ export const TraceSource = Source.discriminator<ITraceSource>(
spanLinksValueExpression: String,
implicitColumnExpression: String,
knownColumnsListExpression: String,
rowKeyExpression: String,
useTextIndexForImplicitColumn: {
type: String,
enum: Object.values(UseTextIndex),
Expand Down
10 changes: 10 additions & 0 deletions packages/api/src/routers/external-api/v2/sources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,11 @@ function formatExternalSource(source: SourceDocument) {
* description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns.
* nullable: true
* example: Timestamp, Body, ServiceName
* rowKeyExpression:
* type: string
* description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.
* nullable: true
* example: toDate(Timestamp), ServiceName, Timestamp, _block_number, _block_offset
* useTextIndexForImplicitColumn:
* type: string
* enum: [auto, enabled, disabled]
Expand Down Expand Up @@ -607,6 +612,11 @@ function formatExternalSource(source: SourceDocument) {
* description: For Distributed table sources whose target tables have non-matching column sets. A list of columns supported across all target tables, used instead of SELECT * when fetching full row data. Leave blank to select all columns.
* nullable: true
* example: Timestamp, Body, ServiceName
* rowKeyExpression:
* type: string
* description: Comma-separated expressions that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only. Leave blank to detect keys from table metadata.
* nullable: true
* example: toDate(Timestamp), SpanName, Timestamp, _block_number, _block_offset
* useTextIndexForImplicitColumn:
* type: string
* enum: [auto, enabled, disabled]
Expand Down
49 changes: 41 additions & 8 deletions packages/app/src/components/DBRowTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ import {
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
import {
BuilderChartConfigWithDateRange,
isLogSource,
isTraceSource,
SelectList,
TSource,
} from '@hyperdx/common-utils/dist/types';
Expand Down Expand Up @@ -1576,15 +1578,27 @@ function getSelectLength(select: SelectList): number {
}
}

function getRowKeyExpression(source: TSource | undefined): string | undefined {
Comment thread
karl-power marked this conversation as resolved.
Resolved
return source && (isLogSource(source) || isTraceSource(source))
? source.rowKeyExpression?.trim() || undefined
: undefined;
}

export function useConfigWithAdditionalSelect(
config: BuilderChartConfigWithDateRange,
sourceId?: string,
rowKeyExpression?: string,
) {
const { data: tableMetadata } = useTableMetadata({
databaseName: config.from.databaseName,
tableName: config.from.tableName,
connectionId: config.connection,
});
const hasRowKeyOverride = !!sourceId && !!rowKeyExpression;

const { data: tableMetadata } = useTableMetadata(
{
databaseName: config.from.databaseName,
tableName: config.from.tableName,
connectionId: config.connection,
},
hasRowKeyOverride ? { enabled: false } : undefined,
);

// Only check for row-ID columns for row-level queries (sourceId present).
// Skip for aggregate queries (e.g. patterns) where extra keys are irrelevant.
Expand All @@ -1594,12 +1608,13 @@ export function useConfigWithAdditionalSelect(
tableName: config.from.tableName,
connectionId: config.connection,
},
{ enabled: !!sourceId },
{ enabled: !!sourceId && !hasRowKeyOverride },
);

const engineFull = tableMetadata?.engine_full ?? '';
const needsMergeTreeSettings =
!!sourceId &&
!hasRowKeyOverride &&
!!tableMetadata?.engine?.includes('MergeTree') &&
omitsBlockColumnSetting(engineFull);

Expand All @@ -1612,6 +1627,19 @@ export function useConfigWithAdditionalSelect(
const partitionKey = tableMetadata?.partition_key;

return useMemo(() => {
if (sourceId && rowKeyExpression) {
const { select, additionalKeysLength } = appendSelectWithAdditionalKeys(
Comment thread
karl-power marked this conversation as resolved.
Resolved
config.select,
rowKeyExpression,
'',
);
const rowKeyColumns = new Set([
Comment thread
karl-power marked this conversation as resolved.
Resolved
...splitAndTrimWithBracket(rowKeyExpression),
...extractColumnReferencesFromKey(rowKeyExpression),
]);
return { ...config, select, additionalKeysLength, rowKeyColumns };
}

// Wait for settings to avoid querying twice.
if (
primaryKey == null ||
Expand Down Expand Up @@ -1674,6 +1702,7 @@ export function useConfigWithAdditionalSelect(
columns,
mergeTreeSettings,
sourceId,
rowKeyExpression,
]);
}

Expand Down Expand Up @@ -1821,7 +1850,12 @@ function DBSqlRowTableComponent({
return base;
}, [me, config, orderByArray]);

const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj, sourceId);
const { data: source } = useSource({ id: sourceId });
Comment thread
karl-power marked this conversation as resolved.
Resolved
const mergedConfig = useConfigWithAdditionalSelect(
mergedConfigObj,
sourceId,
getRowKeyExpression(source),
);

const { data, fetchNextPage, hasNextPage, isFetching, isError, error } =
useOffsetPaginatedQuery(mergedConfig ?? config, {
Expand Down Expand Up @@ -1943,7 +1977,6 @@ function DBSqlRowTableComponent({
}
}, [data?.meta, onResolvedColumnsChange]);

const { data: source } = useSource({ id: sourceId });
const denoised = useDenoisedRows({
config,
source,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
DEFAULT_DATABASE,
EVENT_NAME_EXPRESSION_HELP_TEXT,
KNOWN_COLUMNS_EXPRESSION_HELP_TEXT,
ROW_KEY_EXPRESSION_HELP_TEXT,
SERVICE_VERSION_EXPRESSION_HELP_TEXT,
SEVERITY_NUMBER_EXPRESSION_HELP_TEXT,
} from './constants';
Expand Down Expand Up @@ -285,6 +286,22 @@ export function LogTableModelForm(props: TableModelProps) {
disableKeywordAutocomplete
/>
</FormRow>
<FormRow
label="Row key expression"
helpText={ROW_KEY_EXPRESSION_HELP_TEXT}
>
<SQLInlineEditorControlled
tableConnection={{
databaseName,
tableName,
connectionId,
}}
control={control}
name="rowKeyExpression"
placeholder="ServiceName, Timestamp, _block_number, _block_offset"
disableKeywordAutocomplete
/>
</FormRow>
<UseTextIndexFormRow control={control} />
<Separator size="xs" />
<HighlightedAttributeExpressionsFormRow
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { useColumns } from '@/hooks/useMetadata';
import {
DEFAULT_DATABASE,
KNOWN_COLUMNS_EXPRESSION_HELP_TEXT,
ROW_KEY_EXPRESSION_HELP_TEXT,
SERVICE_VERSION_EXPRESSION_HELP_TEXT,
} from './constants';
import { ExpressionFormRow } from './ExpressionFormRow';
Expand Down Expand Up @@ -379,6 +380,22 @@ export function TraceTableModelForm(props: TableModelProps) {
disableKeywordAutocomplete
/>
</FormRow>
<FormRow
label="Row key expression"
helpText={ROW_KEY_EXPRESSION_HELP_TEXT}
>
<SQLInlineEditorControlled
tableConnection={{
databaseName,
tableName,
connectionId,
}}
control={control}
name="rowKeyExpression"
placeholder="ServiceName, Timestamp, _block_number, _block_offset"
disableKeywordAutocomplete
/>
</FormRow>
<UseTextIndexFormRow control={control} />
<FormRow
label={'Displayed Timestamp Column'}
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/Sources/SourceForm/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ export const SOURCE_KIND_LABELS: Record<SourceKind, string> = {
export const KNOWN_COLUMNS_EXPRESSION_HELP_TEXT =
'For Distributed table sources whose target tables have non-matching column sets. Provide a list of columns supported across all target tables; it is used instead of SELECT * when fetching full row data (e.g. the row side panel). Leave blank to select all columns. This should be a comma-separated list of column names - do not include non-column expressions or aliases.';

export const ROW_KEY_EXPRESSION_HELP_TEXT =
'Comma-separated columns that together identify one row, e.g. the partition key, primary key, _block_number and _block_offset. When set, row details are looked up by these columns only.';

export const SERVICE_VERSION_EXPRESSION_HELP_TEXT =
"Identifies the running release of a service. Defaults to the OpenTelemetry service.version resource attribute. Point it elsewhere if your release identifier lives in another attribute - under GitOps it is often the container image tag. If services in this table use different attributes, fall back across them: coalesce(nullIf(ResourceAttributes['service.version'], ''), nullIf(ResourceAttributes['container.image.tag'], '')).";

Expand Down
81 changes: 81 additions & 0 deletions packages/app/src/components/__tests__/DBRowTable.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1198,4 +1198,85 @@ describe('useConfigWithAdditionalSelect', () => {
'Timestamp,ServiceName,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_offset',
);
});

describe('with a row key expression', () => {
const metadataLoading = () => {
mockUseTableMetadata.mockReturnValue({ data: undefined });
mockUseColumns.mockReturnValue({ data: undefined });
mockUseMergeTreeSettings.mockReturnValue({
data: undefined,
isLoading: true,
});
};

it('restricts the row lookup to the listed columns', () => {
metadataLoading();

const { result } = renderHook(() =>
useConfigWithAdditionalSelect(
config,
'source-id',
'ServiceName, Timestamp, _block_number, _block_offset',
),
);

expect(result.current?.select).toBe(
'Timestamp,ServiceName,Body,_block_number,_block_offset',
);
expect(result.current?.additionalKeysLength).toBe(2);
expect(result.current?.rowKeyColumns).toEqual(
new Set(['ServiceName', 'Timestamp', '_block_number', '_block_offset']),
);
expect(result.current?.rowKeyColumns?.has('Body')).toBe(false);
});

it('returns a config without waiting for table metadata or settings', () => {
metadataLoading();

const { result } = renderHook(() =>
useConfigWithAdditionalSelect(
config,
'source-id',
'Timestamp, _block_number, _block_offset',
),
);

expect(mockUseTableMetadata).toHaveBeenCalledWith(expect.anything(), {
enabled: false,
});
expect(mockUseColumns).toHaveBeenCalledWith(expect.anything(), {
enabled: false,
});
expect(mockUseMergeTreeSettings).toHaveBeenCalledWith('conn', {
enabled: false,
});
expect(result.current).toBeDefined();
});

it('does not split inside function calls', () => {
metadataLoading();

const { result } = renderHook(() =>
useConfigWithAdditionalSelect(
config,
'source-id',
'toYYYYMM(event_date), service, _block_number, _block_offset',
),
);

expect(result.current?.select).toBe(
'Timestamp,ServiceName,Body,event_date,service,_block_number,_block_offset,toYYYYMM(event_date)',
);
expect(result.current?.additionalKeysLength).toBe(5);
expect(result.current?.rowKeyColumns).toEqual(
new Set([
'toYYYYMM(event_date)',
'service',
'_block_number',
'_block_offset',
'event_date',
]),
);
});
});
});
Loading
Loading