Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/sql-query-dashboard-filter-schema.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/api': patch
'@hyperdx/app': patch
---

feat: add the SQL query dashboard filter type to the dashboard schema

A `SQL_QUERY` filter stores a ClickHouse connection and a SQL template whose first column fills the filter's dropdown. It is variable-only. Validation flags templates that use `$__sourceTable` or `$__filters`, or whose macros and variable references can't expand. The filter is not yet exposed through the external API or MCP, and has no editor or values lookup yet.
8 changes: 6 additions & 2 deletions packages/api/src/utils/externalApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
isFilterBroadcastEnabled,
isFilterRequired,
isFilterVariableEnabled,
isSqlQueryFilter,
} from '@hyperdx/common-utils/dist/filters';
import {
AlertErrorType,
Expand Down Expand Up @@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig(
}

/** Filter types the external API and MCP don't expose yet. */
type InternalOnlyDashboardFilter = Extract<DashboardFilter, { type: 'ADHOC' }>;
type InternalOnlyDashboardFilter = Extract<
DashboardFilter,
{ type: 'ADHOC' | 'SQL_QUERY' }
>;
export type ExternalCompatibleDashboardFilter = Exclude<
DashboardFilter,
InternalOnlyDashboardFilter
Expand All @@ -253,7 +257,7 @@ export function getExternalCompatibleFilters(
): ExternalCompatibleDashboardFilter[] {
return (filters ?? []).filter(
(filter): filter is ExternalCompatibleDashboardFilter =>
!isAdhocFilter(filter),
!isAdhocFilter(filter) && !isSqlQueryFilter(filter),
Comment thread
karl-power marked this conversation as resolved.
Resolved
);
}

Expand Down
152 changes: 126 additions & 26 deletions packages/app/src/DBDashboardImportPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
isAdhocFilter,
isPrometheusLabelFilter,
isQueryExpressionFilter,
isSqlQueryFilter,
QUERY_EXPRESSION_FILTER_SOURCE_KINDS,
type SavedFilterValueIssue,
type SavedQueryIssue,
Expand Down Expand Up @@ -337,10 +338,17 @@ function FileSelection({
details={
<Stack gap={0}>
{filterQueryWarnings.map(warning => (
<Text key={warning.filterId} c="yellow">
{warning.filterName}: invalid {warning.language} query:{' '}
{warning.where}
</Text>
<Stack key={warning.filterId} gap={0}>
<Text c="yellow">
{warning.filterName}: invalid {warning.language} query:{' '}
{warning.where}
</Text>
{warning.detail && (
Comment thread
karl-power marked this conversation as resolved.
Resolved
<Text c="yellow" size="sm">
{warning.detail}
</Text>
)}
</Stack>
))}
</Stack>
}
Expand Down Expand Up @@ -458,6 +466,8 @@ export function buildMappingFormSchema(
connectionMappings: z.array(MappingValueSchema),
/** A list of filter source mappings, ordered by input filter index */
filterSourceMappings: z.array(MappingValueSchema).optional(),
/** A list of filter connection mappings, ordered by input filter index. Only applicable for SQL query filters */
filterConnectionMappings: z.array(MappingValueSchema).optional(),
/**
* Per-filter applies-to source mappings. Each entry is an array of mapped
* source IDs in the same order as the filter's `appliesToSourceIds` names
Expand Down Expand Up @@ -546,6 +556,17 @@ export function buildMappingFormSchema(
return;
}

if (isSqlQueryFilter(filter)) {
if (!data.filterConnectionMappings?.[idx]) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['filterConnectionMappings', idx],
message: 'Select a connection for this filter',
});
}
return;
}

if (!isSourceBackedTemplateFilter(filter)) return;
const mappedSourceId = data.filterSourceMappings?.[idx];
if (!mappedSourceId) {
Expand Down Expand Up @@ -601,6 +622,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
tileSourceMappings: input.tiles.map(() => ''),
connectionMappings: input.tiles.map(() => ''),
filterSourceMappings: input.filters?.map(() => '') ?? [],
filterConnectionMappings: input.filters?.map(() => '') ?? [],
filterAppliesToSourceMappings: input.filters?.map(() => []) ?? [],
filterAdhocSourceMappings: input.filters?.map(() => []) ?? [],
onClickSourceMappings: input.tiles.map(() => ''),
Expand Down Expand Up @@ -631,15 +653,20 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
return match?.id || '';
});

const findConnectionIdByName = (name: string) =>
connections.find(conn => conn.name.toLowerCase() === name.toLowerCase())
?.id || '';

const connectionMappings = input.tiles.map(tile => {
const config = tile.config as SavedChartConfig;
if (!isRawSqlSavedChartConfig(config)) return '';
const match = connections.find(
conn => conn.name.toLowerCase() === config.connection.toLowerCase(),
);
return match?.id || '';
return findConnectionIdByName(config.connection);
});

const filterConnectionMappings = input.filters?.map(filter =>
isSqlQueryFilter(filter) ? findConnectionIdByName(filter.connection) : '',
);

const filterSourceMappings = input.filters?.map(filter => {
const sourceName = sourceBackedTemplateFilter(filter)?.source;
if (sourceName == null) return '';
Expand Down Expand Up @@ -698,6 +725,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
setValue('tileSourceMappings', tileSourceMappings);
setValue('connectionMappings', connectionMappings);
setValue('filterSourceMappings', filterSourceMappings);
setValue('filterConnectionMappings', filterConnectionMappings);
multiMappingSlotsRef.current.clear();
filterAppliesToSourceMappings?.forEach((mapping, idx) => {
setValue(`filterAppliesToSourceMappings.${idx}`, mapping);
Expand All @@ -720,6 +748,10 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
name: 'onClickSourceMappings',
});
const connectionMappings = useWatch({ control, name: 'connectionMappings' });
const filterConnectionMappings = useWatch({
control,
name: 'filterConnectionMappings',
});
const onClickDashboardMappings = useWatch({
control,
name: 'onClickDashboardMappings',
Expand All @@ -728,6 +760,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
const prevFilterSourceMappingsRef = useRef(filterSourceMappings);
const prevOnClickSourceMappingsRef = useRef(onClickSourceMappings);
const prevConnectionMappingsRef = useRef(connectionMappings);
const prevFilterConnectionMappingsRef = useRef(filterConnectionMappings);
const prevOnClickDashboardMappingsRef = useRef(onClickDashboardMappings);

// Propagate source mapping changes to other tiles/filters/onClicks with the
Expand Down Expand Up @@ -883,26 +916,49 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
sources,
]);

// Propagate connection mapping changes to other RawSQL tiles with the same input connection
// Propagate connection mapping changes to other RawSQL tiles and SQL query
Comment thread
karl-power marked this conversation as resolved.
Resolved
// filters with the same input connection
useEffect(() => {
if (isUpdatingRef.current) return;
if (!connectionMappings || !input.tiles) return;

const changedIdx = connectionMappings.findIndex(
(mapping, idx) =>
mapping !== prevConnectionMappingsRef.current?.[idx] &&
getFieldState(`connectionMappings.${idx}`).isDirty,
);
if (changedIdx === -1) return;
if (!input.tiles) return;

prevConnectionMappingsRef.current = connectionMappings;
let inputConnectionName: string | undefined;
let connectionId = '';

const tileIdx =
connectionMappings?.findIndex(
(mapping, idx) =>
mapping !== prevConnectionMappingsRef.current?.[idx] &&
getFieldState(`connectionMappings.${idx}`).isDirty,
) ?? -1;
if (connectionMappings && tileIdx !== -1) {
prevConnectionMappingsRef.current = connectionMappings;
const config = input.tiles[tileIdx]?.config as
| SavedChartConfig
| undefined;
if (config && isRawSqlSavedChartConfig(config)) {
inputConnectionName = config.connection;
connectionId = connectionMappings[tileIdx] ?? '';
}
}

const inputTile = input.tiles[changedIdx];
const inputTileConfig = inputTile?.config as SavedChartConfig | undefined;
if (!inputTileConfig || !isRawSqlSavedChartConfig(inputTileConfig)) return;
if (inputConnectionName == null && filterConnectionMappings) {
const filterIdx = filterConnectionMappings.findIndex(
(mapping, idx) =>
mapping !== prevFilterConnectionMappingsRef.current?.[idx] &&
getFieldState(`filterConnectionMappings.${idx}`).isDirty,
);
if (filterIdx !== -1) {
prevFilterConnectionMappingsRef.current = filterConnectionMappings;
const filter = input.filters?.[filterIdx];
if (filter && isSqlQueryFilter(filter)) {
inputConnectionName = filter.connection;
connectionId = filterConnectionMappings[filterIdx] ?? '';
}
}
}

const connectionId = connectionMappings[changedIdx] ?? '';
const inputTileConnection = inputTileConfig.connection;
if (inputConnectionName == null) return;

const keysForTilesWithMatchingConnection = input.tiles
.map((tile, index) => ({
Expand All @@ -912,18 +968,38 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
.filter(
tile =>
isRawSqlSavedChartConfig(tile.config) &&
tile.config.connection === inputTileConnection,
tile.config.connection === inputConnectionName,
)
.map(({ index }) => `connectionMappings.${index}` as const);

const keysForFiltersWithMatchingConnection =
input.filters
?.map((filter, index) => ({ filter, index }))
.filter(
({ filter }) =>
isSqlQueryFilter(filter) &&
filter.connection === inputConnectionName,
)
.map(({ index }) => `filterConnectionMappings.${index}` as const) ?? [];

isUpdatingRef.current = true;
for (const key of keysForTilesWithMatchingConnection) {
for (const key of [
...keysForTilesWithMatchingConnection,
...keysForFiltersWithMatchingConnection,
]) {
if (!getFieldState(key).isDirty) {
setValue(key, connectionId, { shouldValidate: true });
}
}
isUpdatingRef.current = false;
}, [connectionMappings, input.tiles, getFieldState, setValue]);
}, [
connectionMappings,
filterConnectionMappings,
input.tiles,
input.filters,
getFieldState,
setValue,
]);

// Propagate dashboard mapping changes to other tiles whose onClick targets
// the same input dashboard. Dashboard-type onClicks in a template store the
Expand Down Expand Up @@ -1054,6 +1130,12 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
appliesToSourceIds,
};
}
if (isSqlQueryFilter(filter)) {
const connection = findConnection(
data.filterConnectionMappings?.[idx],
);
return { ...filter, connection: connection!.id };
}
const source = findSource(data.filterSourceMappings?.[idx]);
return {
...filter,
Expand Down Expand Up @@ -1256,6 +1338,24 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
<Table.Td />
</Table.Tr>
)}
{isSqlQueryFilter(filter) && (
<Table.Tr>
<Table.Td>{filter.name} (Filter)</Table.Td>
<Table.Td>Data connection</Table.Td>
<Table.Td>{filter.connection}</Table.Td>
<Table.Td>
<SelectControlled
control={control}
name={`filterConnectionMappings.${i}`}
data={connections?.map(conn => ({
value: conn.id,
label: conn.name,
}))}
placeholder="Select a connection"
/>
</Table.Td>
</Table.Tr>
)}
{isAdhocFilter(filter) && (
<Table.Tr>
<Table.Td>{filter.name} (Filter)</Table.Td>
Expand Down
56 changes: 56 additions & 0 deletions packages/app/src/__tests__/DBDashboardImportPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,24 @@ const unresolvedConnectionTemplate = DashboardTemplateSchema.parse({
],
});

const sqlQueryFilterTemplate = (connection: string) =>
DashboardTemplateSchema.parse({
version: '0.1.0',
name: 'SQL Query Filter',
tiles: [builderTile('tile-number', 'number', 'Metrics', 0)],
filters: [
{
id: 'filter-env',
type: 'SQL_QUERY',
name: 'Environment',
connection,
sqlTemplate: 'SELECT DISTINCT env FROM envs',
isBroadcastEnabled: false,
isVariableEnabled: true,
},
],
});

// A markdown tile renders static content, so its source is not worth blocking
// the import over even when the template names one that no longer exists.
const markdownStaleSourceTemplate = DashboardTemplateSchema.parse({
Expand Down Expand Up @@ -389,6 +407,44 @@ describe('Dashboard import - PromQL label filter source', () => {
});
});

describe('Dashboard import - SQL query filter connection', () => {
it('auto-maps the connection by name and imports it as an id', async () => {
renderWithMantine(<Mapping input={sqlQueryFilterTemplate('default')} />);

const finish = await screen.findByRole('button', {
name: /finish import/i,
});
await act(async () => {
fireEvent.click(finish);
});

await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1));
const payload = mockMutateAsync.mock.calls[0][0];
expect(DashboardWithoutIdSchema.safeParse(payload).success).toBe(true);
expect(payload.filters[0]).toMatchObject({
type: 'SQL_QUERY',
connection: 'conn-default',
});
});

it('blocks the import and flags the filter row when its connection is unmapped', async () => {
renderWithMantine(<Mapping input={sqlQueryFilterTemplate(MISSING)} />);

await act(async () => {
fireEvent.click(
await screen.findByRole('button', { name: /finish import/i }),
);
});

await waitFor(() =>
expect(
screen.getByText('Select a connection for this filter'),
).toBeInTheDocument(),
);
expect(mockMutateAsync).not.toHaveBeenCalled();
});
});

describe('Dashboard import - queried filter source is required', () => {
it('blocks the import and flags the filter row when its source is unmapped', async () => {
renderWithMantine(<Mapping input={unresolvedFilterSourceTemplate} />);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
IconPencil,
IconRefresh,
IconSearch,
IconSql,
IconTrash,
} from '@tabler/icons-react';

Expand Down Expand Up @@ -91,6 +92,12 @@ function getValuesAttribute(
tooltip: 'Sources the condition keys and values come from',
label: getSourceNames(filter.sources, sources),
};
case 'SQL_QUERY':
Comment thread
karl-power marked this conversation as resolved.
Resolved
return {
icon: <IconSql size={14} />,
tooltip: 'The dropdown values come from a SQL query',
label: 'SQL query',
};
default:
filter satisfies never; // exhaustive check
return { icon: <IconSearch size={14} />, label: '' };
Expand Down
Loading
Loading