From 7c20a9adbdb1edfb32a24a5682a97ea110e62aa1 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:50:21 +0000 Subject: [PATCH 1/5] add an inspector of the built-in agent: every thread of a conversation with its whole log, its jobs and what the model was sent --- packages/api/src/harness/inference.ts | 11 +- .../api/src/harness/inspect-route.test.ts | 252 ++++++++++++++ packages/api/src/harness/inspect-route.ts | 260 ++++++++++++++ packages/api/src/harness/persona-response.ts | 26 ++ packages/api/src/harness/route.ts | 25 +- packages/api/src/server/create-api-router.ts | 5 + packages/db/src/harness.test.ts | 6 +- packages/db/src/harness.ts | 56 ++- packages/utils/src/experiments.ts | 3 + packages/web/src/client/harness-client.ts | 2 +- packages/web/src/client/inspect-client.ts | 148 ++++++++ packages/web/src/components/inspect-log.tsx | 196 +++++++++++ .../web/src/components/inspect-request.tsx | 207 +++++++++++ .../web/src/components/inspect-threads.tsx | 250 +++++++++++++ packages/web/src/lib/inspect.test.ts | 187 ++++++++++ packages/web/src/lib/inspect.ts | 144 ++++++++ packages/web/src/loaders.ts | 25 ++ packages/web/src/pages/inspect-page.test.tsx | 329 ++++++++++++++++++ packages/web/src/pages/inspect-page.tsx | 217 ++++++++++++ packages/web/src/router.tsx | 7 + 20 files changed, 2320 insertions(+), 36 deletions(-) create mode 100644 packages/api/src/harness/inspect-route.test.ts create mode 100644 packages/api/src/harness/inspect-route.ts create mode 100644 packages/api/src/harness/persona-response.ts create mode 100644 packages/web/src/client/inspect-client.ts create mode 100644 packages/web/src/components/inspect-log.tsx create mode 100644 packages/web/src/components/inspect-request.tsx create mode 100644 packages/web/src/components/inspect-threads.tsx create mode 100644 packages/web/src/lib/inspect.test.ts create mode 100644 packages/web/src/lib/inspect.ts create mode 100644 packages/web/src/pages/inspect-page.test.tsx create mode 100644 packages/web/src/pages/inspect-page.tsx diff --git a/packages/api/src/harness/inference.ts b/packages/api/src/harness/inference.ts index 45b463f9..0e4fcbd4 100644 --- a/packages/api/src/harness/inference.ts +++ b/packages/api/src/harness/inference.ts @@ -50,14 +50,14 @@ const SUB_FLOW = // How a long log is summarized, by the same model, for itself: a conversation's, or a // sub-agent's, whose one message from the person is its brief. -const SUMMARIZE = +export const SUMMARIZE = 'You are summarizing a conversation between a person and their agent, written out below with the calls the agent made and what they answered, so that the agent can go on from the summary in place of the conversation so far. Write what the agent needs to go on as if it remembered: what the person asked for and what they said they want in general; what was done and found, with the names, ids, dates and numbers that matter; what the person was told; what is still open, promised or planned, such as what to do after a sleep or when a sub-agent reports. Leave out what no longer matters. Plain prose, from the agent’s side, no title.'; -const SUMMARIZE_NOW = 'Write the summary now.'; +export const SUMMARIZE_NOW = 'Write the summary now.'; // How many answers the model may give in a row without the person saying anything, calls, // reports and wake-ups included, before it stops and waits for them; a report that comes after // the stop is acted on, and the count starts over. -const MAX_TURNS_ALONE = 30; +export const MAX_TURNS_ALONE = 30; // How often a model that can't be reached is tried, and how long to wait between tries. const MAX_MODEL_ATTEMPTS = 3; @@ -80,9 +80,8 @@ function isStop(entry: EntryRow): boolean { return entry.kind === 'assistant' && (entry.text === TOO_LONG || entry.text === TOO_LONG_SUB); } -// How many answers the model has given since the person last spoke, or since it last stopped for -// them. -function turnsAlone(entries: EntryRow[]): number { +/** How many answers the model has given since the person last spoke, or since it last stopped for them. */ +export function turnsAlone(entries: EntryRow[]): number { const since = entries.findLastIndex((entry) => entry.kind === 'person' || isStop(entry)); return entries.slice(since + 1).filter((entry) => entry.kind === 'assistant').length; } diff --git a/packages/api/src/harness/inspect-route.test.ts b/packages/api/src/harness/inspect-route.test.ts new file mode 100644 index 00000000..2d479876 --- /dev/null +++ b/packages/api/src/harness/inspect-route.test.ts @@ -0,0 +1,252 @@ +import {beforeEach, describe, expect, mock, test} from 'bun:test'; +import {Err, Ok} from 'ts-results-es'; + +import {ApiErr} from '@proxy/utils'; + +mock.module('../utils/env', () => ({env: {NODE_ENV: 'test', APP_URL: 'http://localhost:5173'}})); + +const agentInstructions = mock(async (_agent: unknown, flow: string) => Ok(`You are signed in.\n\n${flow}`)); +mock.module('../mcp/tools', () => ({agentInstructions, MCP_TOOLS: []})); + +const delegate = {name: 'delegate', description: 'Delegates', parameters: {}}; +const read = {name: 'read', description: 'Reads', parameters: {}}; +mock.module('./tool-surface', () => ({toolsFor: (candidate: {parentThreadId: string | null}) => (candidate.parentThreadId === null ? [delegate] : [read])})); + +const getPersona = mock(); +const getMemberThread = mock(); +const listSubAgentThreads = mock(); +const listThreadJobs = mock(); +const listJobEvents = mock(); +const listEntriesWithPending = mock(); +const listEntries = mock(); +const getEntry = mock(); +const getJobThread = mock(); +// What the route reads, and what the inference module it takes its constants from imports. +mock.module('@proxy/db/harness', () => ({ + getPersona, + getMemberThread, + listSubAgentThreads, + listThreadJobs, + listJobEvents, + listEntriesWithPending, + listEntries, + getEntry, + getJobThread, + appendSummary: mock(), + completeInference: mock(), + finishJob: mock(), + getThreadAgent: mock(), + requeueJob: mock(), +})); + +const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'harness_debug']}; +const requireViewer = mock(async () => Ok(viewer)); +mock.module('../utils/user-org', () => ({requireViewer})); + +const now = new Date('2026-10-21T10:00:00Z'); +const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; +const persona = {id: 'persona-1', name: 'Finance', look, color: 'blue', createdById: 'user-1'}; +const thread = { + id: 'thread-1', + orgId: 'org-1', + agentId: 'login-1', + personaId: 'persona-1', + userId: 'user-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, +}; +const sub = {...thread, id: 'thread-2', parentThreadId: 'thread-1', parentCallId: 'c1', title: 'Find the invoices', taskId: 'task-2'}; + +type Kind = 'person' | 'assistant' | 'tool_result' | 'event' | 'summary'; + +function entry( + seq: number, + threadId: string, + kind: Kind, + text: string, + extra: Partial<{toolCalls: Array<{id: string; name: string; arguments: string}>; callId: string; inputTokens: number; coversSeq: number; system: string; jobId: string; pending: boolean}> = {}, +) { + return { + id: `e${seq}`, + seq, + threadId, + kind, + pending: extra.pending ?? false, + text, + toolCalls: extra.toolCalls ?? null, + callId: extra.callId ?? null, + isError: false, + inputTokens: extra.inputTokens ?? null, + outputTokens: null, + coversSeq: extra.coversSeq ?? null, + system: extra.system ?? null, + jobId: extra.jobId ?? null, + createdAt: now, + }; +} + +// The person asked, the agent delegated, and the sub-agent has read once and is waiting on it. +const conversation = [ + entry(1, 'thread-1', 'person', 'Find my invoices'), + entry(2, 'thread-1', 'assistant', '', { + toolCalls: [{id: 'c1', name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}], + inputTokens: 1200, + system: 'You are signed in.', + jobId: 'job-1', + }), + entry(3, 'thread-1', 'tool_result', 'Started the sub-agent “Find the invoices”.', {callId: 'c1', jobId: 'job-2'}), + entry(4, 'thread-1', 'assistant', 'Looking.', {inputTokens: 1300, system: 'You are signed in.', jobId: 'job-3'}), + entry(7, 'thread-1', 'person', 'Thanks', {pending: true}), +]; +const subLog = [ + entry(5, 'thread-2', 'person', 'Find them'), + entry(6, 'thread-2', 'assistant', '', {toolCalls: [{id: 'c2', name: 'read', arguments: '{}'}], inputTokens: 800, system: 'You are a sub-agent.', jobId: 'job-4'}), +]; +const jobs = [ + { + id: 'job-1', + threadId: 'thread-1', + kind: 'inference' as const, + status: 'done' as const, + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 1, + error: null, + createdAt: now, + }, + { + id: 'job-5', + threadId: 'thread-2', + kind: 'tool' as const, + status: 'running' as const, + entryId: 'e6', + callId: 'c2', + runAt: now, + claimedBy: 'host-1', + leaseExpiresAt: now, + attempts: 1, + error: null, + createdAt: now, + }, +]; +const events = [ + {id: 'ev1', jobId: 'job-1', threadId: 'thread-1', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, + {id: 'ev2', jobId: 'job-5', threadId: 'thread-2', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, +]; + +function request(params: Record = {}) { + return {user: {userId: 'user-1', email: 'a@example.com'}, params, body: undefined} as never; +} + +beforeEach(() => { + mock.clearAllMocks(); + requireViewer.mockResolvedValue(Ok(viewer)); + getPersona.mockResolvedValue(Ok(persona)); + getMemberThread.mockResolvedValue(Ok(thread)); + listSubAgentThreads.mockResolvedValue(Ok([sub])); + listThreadJobs.mockResolvedValue(Ok(jobs)); + listJobEvents.mockResolvedValue(Ok(events)); + listEntriesWithPending.mockImplementation(async (threadId: string) => Ok(threadId === 'thread-1' ? conversation : subLog)); + listEntries.mockImplementation(async (threadId: string) => Ok((threadId === 'thread-1' ? conversation : subLog).filter((candidate) => !candidate.pending))); + getEntry.mockImplementation(async (id: string) => { + const found = [...conversation, ...subLog].find((candidate) => candidate.id === id); + return found ? Ok(found) : Err(ApiErr.notFound('entry', id)); + }); + getJobThread.mockImplementation(async (id: string) => Ok(id === 'thread-1' ? thread : id === 'thread-2' ? sub : null)); +}); + +describe('the inspection of a conversation', () => { + test('is refused without the inspector experiment, as the Chat page is without its own', async () => { + const {handleInspectThreadRoute, handleInspectRequestRoute} = await import('./inspect-route'); + requireViewer.mockResolvedValue(Ok({...viewer, experiments: ['harness']})); + expect((await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('forbidden'); + expect((await handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('forbidden'); + }); + + test('is the conversation first, then its sub-agents, each with its whole log, its jobs and their events', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + const result = (await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap(); + expect(result.persona.id).toBe('persona-1'); + expect(result.threads.map((candidate) => candidate.id)).toEqual(['thread-1', 'thread-2']); + expect(listThreadJobs).toHaveBeenCalledWith(['thread-1', 'thread-2']); + expect(listJobEvents).toHaveBeenCalledWith(['thread-1', 'thread-2']); + const [main, agent] = result.threads; + expect(main?.entries.map((candidate) => candidate.kind)).toEqual(['person', 'assistant', 'tool_result', 'assistant', 'person']); + expect(main?.entries.at(-1)?.pending).toBe(true); + expect(main?.entries[1]).toMatchObject({hasSystem: true, jobId: 'job-1', inputTokens: 1200, toolCalls: [{id: 'c1', name: 'delegate', arguments: expect.any(String)}]}); + expect(main?.jobs.map((candidate) => candidate.id)).toEqual(['job-1']); + expect(main?.events.map((candidate) => candidate.id)).toEqual(['ev1']); + expect(agent?.title).toBe('Find the invoices'); + expect(agent?.jobs.map((candidate) => candidate.id)).toEqual(['job-5']); + expect(agent?.jobs[0]).toMatchObject({status: 'running', claimedBy: 'host-1', leaseExpiresAt: now.toISOString()}); + }); + + test('says where each thread stands: the conversation waits for the pending turn to land, the sub-agent for its call', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + const [main, agent] = (await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap().threads; + // The pending turn is not in the log yet: the log ends with an answer, so the model is not due. + expect(main?.state).toMatchObject({ready: false, pendingCalls: [], turnsAlone: 2, maxTurnsAlone: 30, lastInputTokens: 1300, compactAtTokens: 80_000, compactionCoversSeq: null}); + expect(agent?.state).toMatchObject({ready: false, pendingCalls: [{id: 'c2', name: 'read'}], turnsAlone: 1, lastInputTokens: 800}); + }); + + test('is not found before the viewer has a conversation with the persona', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + getMemberThread.mockResolvedValue(Ok(null)); + expect((await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('not_found'); + }); +}); + +describe('the request behind an entry', () => { + test('is the system prompt the answer kept, the log before it as the model read it, and the thread’s tools', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + const result = (await handleInspectRequestRoute(request({entryId: 'e4'}))).unwrap(); + expect(result.kind).toBe('turn'); + expect(result.system).toBe('You are signed in.'); + expect(result.messages).toEqual([ + {role: 'system', content: 'You are signed in.'}, + {role: 'user', content: 'Find my invoices'}, + {role: 'assistant', content: null, tool_calls: [{id: 'c1', type: 'function', function: {name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}}]}, + {role: 'tool', tool_call_id: 'c1', content: 'Started the sub-agent “Find the invoices”.'}, + ]); + expect(result.tools).toEqual([delegate]); + }); + + test('has a sub-agent’s tools on a sub-agent’s thread', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + const result = (await handleInspectRequestRoute(request({entryId: 'e6'}))).unwrap(); + expect(result.messages).toEqual([ + {role: 'system', content: 'You are a sub-agent.'}, + {role: 'user', content: 'Find them'}, + ]); + expect(result.tools).toEqual([read]); + }); + + test('is the transcript of what a summary covers, for the model to summarize', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + getEntry.mockResolvedValue(Ok(entry(5, 'thread-1', 'summary', 'The person asked for invoices.', {coversSeq: 3}))); + const result = (await handleInspectRequestRoute(request({entryId: 'e5'}))).unwrap(); + expect(result.kind).toBe('summary'); + expect(result.tools).toEqual([]); + expect(result.messages[0]?.role).toBe('system'); + const asked = result.messages[1]; + expect(asked?.role).toBe('user'); + expect(asked?.content).toContain('Person:\nFind my invoices'); + expect(asked?.content).toContain('Agent called delegate with'); + expect(asked?.content).not.toContain('Looking.'); + expect(asked?.content).toMatch(/Write the summary now\.$/); + }); + + test('is not found for an entry of someone else’s thread, or one that is neither an answer nor a summary', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + expect((await handleInspectRequestRoute(request({entryId: 'e1'}))).unwrapErr().kind).toBe('not_found'); + getJobThread.mockResolvedValue(Ok({...thread, userId: 'user-2'})); + expect((await handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('not_found'); + }); +}); diff --git a/packages/api/src/harness/inspect-route.ts b/packages/api/src/harness/inspect-route.ts new file mode 100644 index 00000000..3da15cc1 --- /dev/null +++ b/packages/api/src/harness/inspect-route.ts @@ -0,0 +1,260 @@ +import {Err, type Result} from 'ts-results-es'; + +import {ApiErr, type ApiError, Do, hasExperiment, requirePresent} from '@proxy/utils'; +import {pendingToolCalls, readyForInference, type ToolCall} from '@proxy/utils/harness'; +import { + type EntryRow, + getEntry, + getJobThread, + getMemberThread, + getPersona, + type JobDetailRow, + type JobEventRow, + listEntries, + listEntriesWithPending, + listJobEvents, + listSubAgentThreads, + listThreadJobs, + type ThreadRow, +} from '@proxy/db/harness'; + +import type {AuthenticatedRequest} from '../server/middleware/require-auth'; +import {type OrgViewer, requireViewer} from '../utils/user-org'; +import {COMPACT_AT_TOKENS, compactionCut, toMessages, toTranscript} from './context'; +import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from './inference'; +import type {ModelMessage, ModelTool} from './model'; +import {type PersonaResponse, toPersona} from './persona-response'; +import {toolsFor} from './tool-surface'; + +/** + * The inspector's routes, for whoever works on the built-in agent, behind the `harness_debug` + * experiment beside `harness`: the viewer's conversation with a persona as the harness has it, + * every thread of it with its whole log, its jobs and what happened to each, and, for any answer + * of the model, the request it was sent, rebuilt from the log with the functions the harness runs + * (context.ts) and the system prompt the entry kept. The viewer sees their own conversations + * alone, as on the Chat page: a conversation is theirs, whoever they are in the organization. + */ + +/** An entry of a thread's log, whole: every kind, what is pending too, with what the model's answer cost and whether its system prompt was kept, which `GET …/inspect/entries/:entryId/request` gives with the rest of the request. */ +export type InspectedEntry = { + id: string; + seq: number; + kind: EntryRow['kind']; + text: string; + toolCalls: ToolCall[] | null; + callId: string | null; + isError: boolean; + pending: boolean; + inputTokens: number | null; + outputTokens: number | null; + coversSeq: number | null; + hasSystem: boolean; + jobId: string | null; + at: string; +}; + +/** A job as the queue has it, whatever its status. */ +export type InspectedJob = { + id: string; + threadId: string; + kind: JobDetailRow['kind']; + status: JobDetailRow['status']; + entryId: string | null; + callId: string | null; + runAt: string; + claimedBy: string | null; + leaseExpiresAt: string | null; + attempts: number; + error: string | null; + createdAt: string; +}; + +/** What happened to a job, one row per happening. */ +export type InspectedJobEvent = {id: string; jobId: string; threadId: string; kind: JobEventRow['kind']; workerId: string | null; error: string | null; at: string}; + +/** Where the thread stands for the harness: whether its log is ready for the model and what holds it back, how many steps the model took alone against the limit, and how close the log is to being summarized. */ +export type ThreadState = { + ready: boolean; + pendingCalls: ToolCall[]; + turnsAlone: number; + maxTurnsAlone: number; + lastInputTokens: number; + compactAtTokens: number; + // The seq of the last entry a summary made now would cover; null when the log needs none. + compactionCoversSeq: number | null; +}; + +/** A thread of the conversation: the conversation itself, or a sub-agent started from it. */ +export type InspectedThread = { + id: string; + parentThreadId: string | null; + parentCallId: string | null; + title: string | null; + taskId: string | null; + finishedAt: string | null; + createdAt: string; + entries: InspectedEntry[]; + jobs: InspectedJob[]; + events: InspectedJobEvent[]; + state: ThreadState; +}; + +/** The viewer's conversation with a persona, whole: the conversation's thread first, then every sub-agent started from it, oldest first. */ +export type InspectionResponse = {persona: PersonaResponse; now: string; threads: InspectedThread[]}; + +/** What the model was sent for an entry: the system prompt as kept (null where the answer was written without the model), the messages and the tools, rebuilt from the log as it was. */ +export type RequestResponse = {kind: 'turn' | 'summary'; system: string | null; messages: ModelMessage[]; tools: ModelTool[]}; + +function toInspectedEntry(entry: EntryRow): InspectedEntry { + return { + id: entry.id, + seq: entry.seq, + kind: entry.kind, + text: entry.text, + toolCalls: entry.toolCalls, + callId: entry.callId, + isError: entry.isError, + pending: entry.pending, + inputTokens: entry.inputTokens, + outputTokens: entry.outputTokens, + coversSeq: entry.coversSeq, + hasSystem: entry.system !== null, + jobId: entry.jobId, + at: entry.createdAt.toISOString(), + }; +} + +function toInspectedJob(job: JobDetailRow): InspectedJob { + return { + id: job.id, + threadId: job.threadId, + kind: job.kind, + status: job.status, + entryId: job.entryId, + callId: job.callId, + runAt: job.runAt.toISOString(), + claimedBy: job.claimedBy, + leaseExpiresAt: job.leaseExpiresAt?.toISOString() ?? null, + attempts: job.attempts, + error: job.error, + createdAt: job.createdAt.toISOString(), + }; +} + +function toInspectedEvent(event: JobEventRow): InspectedJobEvent { + return {id: event.id, jobId: event.jobId, threadId: event.threadId, kind: event.kind, workerId: event.workerId, error: event.error, at: event.at.toISOString()}; +} + +// Where the thread stands, from the log the model reads: what is pending is not in it yet. +function stateOf(entries: EntryRow[]): ThreadState { + const log = entries.filter((entry) => !entry.pending); + const cut = compactionCut(log); + const lastAnswer = log.findLast((entry) => entry.kind === 'assistant'); + return { + ready: readyForInference(log), + pendingCalls: pendingToolCalls(log), + turnsAlone: turnsAlone(log), + maxTurnsAlone: MAX_TURNS_ALONE, + lastInputTokens: lastAnswer?.inputTokens ?? 0, + compactAtTokens: COMPACT_AT_TOKENS, + compactionCoversSeq: cut === null ? null : (log[cut - 1]?.seq ?? null), + }; +} + +function requireInspector(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireViewer(request)); + if (!hasExperiment(viewer, 'harness') || !hasExperiment(viewer, 'harness_debug')) { + return $(Err(ApiErr.forbidden())); + } + return viewer; + }); +} + +// A thread the viewer may inspect: one of their own conversations, or a sub-agent started from one. +function mayInspect(viewer: OrgViewer, thread: ThreadRow): boolean { + return thread.orgId === viewer.orgId && thread.userId === viewer.userId; +} + +async function inspectThread(thread: ThreadRow, jobs: JobDetailRow[], events: JobEventRow[]): Promise> { + return Do(async ($) => { + const entries = $(await listEntriesWithPending(thread.id)); + return { + id: thread.id, + parentThreadId: thread.parentThreadId, + parentCallId: thread.parentCallId, + title: thread.title, + taskId: thread.taskId, + finishedAt: thread.finishedAt?.toISOString() ?? null, + createdAt: thread.createdAt.toISOString(), + entries: entries.map(toInspectedEntry), + jobs: jobs.filter((job) => job.threadId === thread.id).map(toInspectedJob), + events: events.filter((event) => event.threadId === thread.id).map(toInspectedEvent), + state: stateOf(entries), + }; + }); +} + +/** `GET /api/orgs/:org/chat/personas/:personaId/inspect`: the viewer's conversation with the persona as the harness has it; not found before they have one. */ +export function handleInspectThreadRoute(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireInspector(request)); + const personaId = request.params.personaId ?? ''; + const persona = $(requirePresent($(await getPersona(viewer.orgId, personaId)), ApiErr.notFound('persona', personaId))); + const conversation = $(requirePresent($(await getMemberThread(persona.id, viewer.userId)), ApiErr.notFound('thread', persona.id))); + const subAgents = $(await listSubAgentThreads(conversation.id)); + const threads = [conversation, ...subAgents]; + const ids = threads.map((thread) => thread.id); + const [jobs, events] = await Promise.all([listThreadJobs(ids), listJobEvents(ids)]); + const allJobs = $(jobs); + const allEvents = $(events); + const inspected: InspectedThread[] = []; + for (const thread of threads) { + inspected.push($(await inspectThread(thread, allJobs, allEvents))); + } + return {persona: toPersona(viewer, persona), now: new Date().toISOString(), threads: inspected}; + }); +} + +// The request behind an assistant entry: the log before it, as the model read it, behind the +// system prompt the entry kept. +function turnRequest(thread: ThreadRow, entry: EntryRow, log: EntryRow[]): RequestResponse { + const before = log.filter((candidate) => candidate.seq < entry.seq); + const system: ModelMessage[] = entry.system === null ? [] : [{role: 'system', content: entry.system}]; + return {kind: 'turn', system: entry.system, messages: [...system, ...toMessages(before, {clearOld: true})], tools: toolsFor(thread)}; +} + +// The request behind a summary entry: the log it covers, written out for the model to summarize. +function summaryRequest(entry: EntryRow, log: EntryRow[]): RequestResponse { + const covered = log.filter((candidate) => candidate.seq <= (entry.coversSeq ?? -1)); + return { + kind: 'summary', + system: SUMMARIZE, + messages: [ + {role: 'system', content: SUMMARIZE}, + {role: 'user', content: `${toTranscript(covered)}\n\n${SUMMARIZE_NOW}`}, + ], + tools: [], + }; +} + +/** `GET /api/orgs/:org/chat/inspect/entries/:entryId/request`: what the model was sent for an assistant or summary entry of one of the viewer's threads, rebuilt from the log; not found for any other entry. */ +export function handleInspectRequestRoute(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireInspector(request)); + const entryId = request.params.entryId ?? ''; + const entry = $(await getEntry(entryId)); + const thread = $(requirePresent($(await getJobThread(entry.threadId)), ApiErr.notFound('thread', entry.threadId))); + if (!mayInspect(viewer, thread)) { + return $(Err(ApiErr.notFound('entry', entryId))); + } + const log = $(await listEntries(thread.id)); + if (entry.kind === 'assistant') { + return turnRequest(thread, entry, log); + } + if (entry.kind === 'summary') { + return summaryRequest(entry, log); + } + return $(Err(ApiErr.notFound('entry', entryId))); + }); +} diff --git a/packages/api/src/harness/persona-response.ts b/packages/api/src/harness/persona-response.ts new file mode 100644 index 00000000..c0fd7edf --- /dev/null +++ b/packages/api/src/harness/persona-response.ts @@ -0,0 +1,26 @@ +import type {LastMessage, PersonaRow} from '@proxy/db/harness'; +import {type Look, preview} from '@proxy/utils/harness'; + +import {manages} from '../utils/manages'; +import type {OrgViewer} from '../utils/user-org'; + +/** A persona as the Chat page lists it, whether the viewer may change it, and the last thing said in their conversation with it, cut to a line; null before anything was said. */ +export type PersonaResponse = {id: string; name: string; look: Look; color: string; mayChange: boolean; lastMessage: {kind: 'person' | 'assistant'; text: string; at: string} | null}; + +/** Whether the viewer may rename, reface or delete the persona: whoever made it, or the organization's owner (utils/manages.ts). */ +export function managesPersona(viewer: OrgViewer, persona: PersonaRow): boolean { + return manages(viewer, {ownerUserId: null, byId: persona.createdById}); +} + +/** A persona for a response, with whether the viewer may change it. */ +export function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): PersonaResponse { + const last = persona.lastMessage ?? null; + return { + id: persona.id, + name: persona.name, + look: persona.look, + color: persona.color, + mayChange: managesPersona(viewer, persona), + lastMessage: last && {kind: last.kind, text: preview(last.text), at: last.createdAt.toISOString()}, + }; +} diff --git a/packages/api/src/harness/route.ts b/packages/api/src/harness/route.ts index 665ed795..f4384153 100644 --- a/packages/api/src/harness/route.ts +++ b/packages/api/src/harness/route.ts @@ -2,7 +2,7 @@ import {Err, type Result} from 'ts-results-es'; import {z} from 'zod'; import {ApiErr, type ApiError, Do, hasExperiment, parseSchema, requirePresent} from '@proxy/utils'; -import {AVATAR_COLORS, type Look, lookSchema, preview} from '@proxy/utils/harness'; +import {AVATAR_COLORS, lookSchema} from '@proxy/utils/harness'; import { appendPersonMessage, createPersona, @@ -12,7 +12,6 @@ import { type EntryRow, getMemberThread, getPersona, - type LastMessage, listOpenJobs, listPersonas, listShownEntries, @@ -23,9 +22,9 @@ import { import {AGENT_NAME_MAX} from '@proxy/integrations'; import type {AuthenticatedRequest} from '../server/middleware/require-auth'; -import {manages} from '../utils/manages'; import {type OrgViewer, requireViewer} from '../utils/user-org'; import {ensureBuiltInAgent} from './built-in-agent'; +import {managesPersona, type PersonaResponse, toPersona} from './persona-response'; /** * The Chat page's routes, under the organization: its personas, which every member sees, each @@ -37,32 +36,12 @@ import {ensureBuiltInAgent} from './built-in-agent'; const MAX_MESSAGE = 20_000; -/** A persona as the page lists it, whether the viewer may change it, and the last thing said in their conversation with it, cut to a line; null before anything was said. */ -export type PersonaResponse = {id: string; name: string; look: Look; color: string; mayChange: boolean; lastMessage: {kind: 'person' | 'assistant'; text: string; at: string} | null}; - /** A turn of the conversation as the page shows it: what the person said, or what the persona answered, which is what went wrong where its turn failed. What the persona did in between is in the activity log, not here. */ export type EntryResponse = {id: string; kind: 'person' | 'assistant'; text: string; isError: boolean; at: string}; /** The member's conversation with a persona: the persona, its turns, and what the persona is up to: busy on it, asleep until a time, or waiting for the person. */ export type ThreadLogResponse = {persona: PersonaResponse; entries: EntryResponse[]; busy: boolean; sleepingUntil: string | null}; -// Whether the viewer may rename, reface or delete the persona: whoever made it, or the organization's owner (utils/manages.ts). -function managesPersona(viewer: OrgViewer, persona: PersonaRow): boolean { - return manages(viewer, {ownerUserId: null, byId: persona.createdById}); -} - -function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): PersonaResponse { - const last = persona.lastMessage ?? null; - return { - id: persona.id, - name: persona.name, - look: persona.look, - color: persona.color, - mayChange: managesPersona(viewer, persona), - lastMessage: last && {kind: last.kind, text: preview(last.text), at: last.createdAt.toISOString()}, - }; -} - // A persona as the person makes or changes it: a name, and a head, a look in one of the colors. const personaSchema = z.object({ name: z.string().trim().pipe(z.string().min(1).max(AGENT_NAME_MAX)), diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index 9b345794..731c5493 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -46,6 +46,7 @@ import { handleShareConnectionRoute, } from '../connections/route'; import {handleFaviconRoute} from '../favicons/route'; +import {handleInspectRequestRoute, handleInspectThreadRoute} from '../harness/inspect-route'; import {handleCreatePersonaRoute, handleDeletePersonaRoute, handleGetThreadRoute, handleListPersonasRoute, handlePostMessageRoute, handleUpdatePersonaRoute} from '../harness/route'; import {handleFillIntentRoute, handlePickIntentRoute} from '../intent/route'; import {handleMcpMethodNotAllowedRoute, handleMcpRoute} from '../mcp/route'; @@ -219,6 +220,10 @@ export function createApiRouter(): express.Router { org.delete('/chat/personas/:personaId', withAuthResult('Delete persona route', handleDeletePersonaRoute)); org.get('/chat/personas/:personaId/thread', withAuthResult('Get chat thread route', handleGetThreadRoute)); org.post('/chat/personas/:personaId/messages', withAuthResult('Post chat message route', handlePostMessageRoute)); + // The inspector, behind the `harness_debug` experiment too: the conversation as the harness has + // it, and what the model was sent for an answer. + org.get('/chat/personas/:personaId/inspect', withAuthResult('Inspect chat thread route', handleInspectThreadRoute)); + org.get('/chat/inspect/entries/:entryId/request', withAuthResult('Inspect chat request route', handleInspectRequestRoute)); // The agent side: its own sign-in and cookie, and every request made for one of the agent's // tasks, checked against the agent's access and logged with the task. diff --git a/packages/db/src/harness.test.ts b/packages/db/src/harness.test.ts index db2eaf3c..3cfa913b 100644 --- a/packages/db/src/harness.test.ts +++ b/packages/db/src/harness.test.ts @@ -118,7 +118,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['assistant', job.id, 'You are signed in.'], ['tool_result', job.id, null], ]); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect(events.map((event) => [event.jobId, event.kind, event.workerId, event.error])).toEqual([ [job.id, 'claimed', 'worker-a', null], [job.id, 'done', 'worker-a', null], @@ -168,7 +168,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['tool_result', tool.id, false], ['assistant', next.id, true], ]); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect( events.slice(2).map((event) => [event.jobId, event.kind, event.workerId, event.error]), ).toEqual([ @@ -188,7 +188,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { throw new Error('nothing to claim'); } unwrap(await finishJob(job.id, 'worker-e')); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect(events.slice(-2).map((event) => [event.jobId, event.kind, event.workerId])).toEqual([ [job.id, 'claimed', 'worker-e'], [job.id, 'done', 'worker-e'], diff --git a/packages/db/src/harness.ts b/packages/db/src/harness.ts index 0a313d25..71fe93fc 100644 --- a/packages/db/src/harness.ts +++ b/packages/db/src/harness.ts @@ -453,17 +453,67 @@ export async function listOpenJobs(threadId: string): Promise> { +/** What happened to the threads' jobs, in order: every claim, requeue, finish and failure. */ +export async function listJobEvents(threadIds: string[]): Promise> { return wrapDb(() => db.agentJobEvent.findMany({ - where: {threadId}, + where: {threadId: {in: threadIds}}, select: jobEventSelect, orderBy: {at: 'asc'}, }), ); } +/** A job with its whole record, for the inspector: beside what the worker reads, why it failed, its lease and when it was made. */ +export type JobDetailRow = JobRow & { + leaseExpiresAt: Date | null; + error: string | null; + createdAt: Date; +}; + +const jobDetailSelect = {...jobSelect, leaseExpiresAt: true, error: true, createdAt: true} as const; + +/** Every job of the threads, whatever its status, oldest first: the queue and its history, for the inspector. */ +export async function listThreadJobs( + threadIds: string[], +): Promise> { + return wrapDb(() => + db.agentJob.findMany({ + where: {threadId: {in: threadIds}}, + select: jobDetailSelect, + orderBy: {createdAt: 'asc'}, + }), + ); +} + +/** The sub-agents started from the thread, done or not, oldest first. */ +export async function listSubAgentThreads( + parentThreadId: string, +): Promise> { + return wrapDb(() => + db.agentThread.findMany({ + where: {parentThreadId}, + select: threadSelect, + orderBy: {createdAt: 'asc'}, + }), + ); +} + +/** The thread's whole log, for the inspector: what the model reads in order, then what is pending, in the order it landed. */ +export async function listEntriesWithPending( + threadId: string, +): Promise> { + return ( + await wrapDb(() => + db.agentEntry.findMany({ + where: {threadId}, + select: entrySelect, + orderBy: [{pending: 'asc'}, {seq: 'asc'}], + }), + ) + ).map((entries) => entries.map(toEntry)); +} + /** A new task for a sub-agent's calls, once the one before was abandoned. */ export async function setThreadTask( threadId: string, diff --git a/packages/utils/src/experiments.ts b/packages/utils/src/experiments.ts index 2b5c326a..bc4b5ab2 100644 --- a/packages/utils/src/experiments.ts +++ b/packages/utils/src/experiments.ts @@ -11,6 +11,9 @@ export const EXPERIMENTS = { // The built-in agent: a conversation with an agent Personal Agent Proxy runs itself, behind its // own gate, on the Chat page. harness: 'Built-in agent', + // The inspector of the built-in agent, for whoever works on it: every thread of a conversation + // with its whole log, its jobs and what the model was sent, on its own page. + harness_debug: 'Built-in agent inspector', } as const satisfies Record; export type Experiment = keyof typeof EXPERIMENTS; diff --git a/packages/web/src/client/harness-client.ts b/packages/web/src/client/harness-client.ts index 70e3c66e..6939c6b9 100644 --- a/packages/web/src/client/harness-client.ts +++ b/packages/web/src/client/harness-client.ts @@ -12,7 +12,7 @@ const personaSchema = z.object({ color: z.enum(AVATAR_COLORS), }); -const chatPersonaSchema = personaSchema.extend({ +export const chatPersonaSchema = personaSchema.extend({ id: z.string(), // Whether the person may rename, reface or delete it: they made it, or own the organization. mayChange: z.boolean(), diff --git a/packages/web/src/client/inspect-client.ts b/packages/web/src/client/inspect-client.ts new file mode 100644 index 00000000..e287fc2c --- /dev/null +++ b/packages/web/src/client/inspect-client.ts @@ -0,0 +1,148 @@ +import {z} from 'zod'; +import {apiRequest, orgApi} from '@/client/request'; +import {chatPersonaSchema} from '@/client/harness-client'; + +// The inspector's api: the person's conversation with a persona as the harness has it, every +// thread of it with its whole log, its jobs and what happened to each; and, for an answer of the +// model, the request it was sent. + +const toolCallSchema = z.object({id: z.string(), name: z.string(), arguments: z.string()}); + +export const entryKinds = ['person', 'assistant', 'tool_result', 'event', 'summary'] as const; + +// An entry of a thread's log, whole: every kind, what is pending too. +const inspectedEntrySchema = z.object({ + id: z.string(), + seq: z.number(), + kind: z.enum(entryKinds), + text: z.string(), + toolCalls: z.array(toolCallSchema).nullable(), + callId: z.string().nullable(), + isError: z.boolean(), + // Landed while the model was answering: not in the log it reads until the answer is. + pending: z.boolean(), + inputTokens: z.number().nullable(), + outputTokens: z.number().nullable(), + // A summary's reach: the seq of the last entry it stands for. + coversSeq: z.number().nullable(), + // Whether the answer kept the system prompt it was sent, which the request route gives. + hasSystem: z.boolean(), + // The job that wrote the entry; null for what the person wrote. + jobId: z.string().nullable(), + at: z.string(), +}); + +export const jobStatuses = ['queued', 'running', 'done', 'failed'] as const; + +// A job as the queue has it, whatever its status. +const inspectedJobSchema = z.object({ + id: z.string(), + threadId: z.string(), + kind: z.enum(['inference', 'tool']), + status: z.enum(jobStatuses), + // A tool job's call: the entry it is in and the id the model gave it. + entryId: z.string().nullable(), + callId: z.string().nullable(), + runAt: z.string(), + claimedBy: z.string().nullable(), + leaseExpiresAt: z.string().nullable(), + attempts: z.number(), + error: z.string().nullable(), + createdAt: z.string(), +}); + +// What happened to a job, one row per happening. +const inspectedJobEventSchema = z.object({ + id: z.string(), + jobId: z.string(), + threadId: z.string(), + kind: z.enum(['claimed', 'requeued', 'done', 'failed']), + workerId: z.string().nullable(), + error: z.string().nullable(), + at: z.string(), +}); + +// Where the thread stands for the harness. +const threadStateSchema = z.object({ + ready: z.boolean(), + pendingCalls: z.array(toolCallSchema), + turnsAlone: z.number(), + maxTurnsAlone: z.number(), + lastInputTokens: z.number(), + compactAtTokens: z.number(), + compactionCoversSeq: z.number().nullable(), +}); + +const inspectedThreadSchema = z.object({ + id: z.string(), + parentThreadId: z.string().nullable(), + parentCallId: z.string().nullable(), + title: z.string().nullable(), + taskId: z.string().nullable(), + finishedAt: z.string().nullable(), + createdAt: z.string(), + entries: z.array(inspectedEntrySchema), + jobs: z.array(inspectedJobSchema), + events: z.array(inspectedJobEventSchema), + state: threadStateSchema, +}); + +const inspectionSchema = z.object({ + persona: chatPersonaSchema, + now: z.string(), + // The conversation first, then every sub-agent started from it, oldest first. + threads: z.array(inspectedThreadSchema), +}); + +// The conversation as the model reads it, in the chat completions form. +const modelMessageSchema = z.discriminatedUnion('role', [ + z.object({role: z.literal('system'), content: z.string()}), + z.object({role: z.literal('user'), content: z.string()}), + z.object({ + role: z.literal('assistant'), + content: z.string().nullable(), + tool_calls: z + .array( + z.object({ + id: z.string(), + type: z.literal('function'), + function: z.object({name: z.string(), arguments: z.string()}), + }), + ) + .optional(), + }), + z.object({role: z.literal('tool'), tool_call_id: z.string(), content: z.string()}), +]); + +const modelToolSchema = z.object({ + name: z.string(), + description: z.string(), + parameters: z.unknown(), +}); + +const requestSchema = z.object({ + kind: z.enum(['turn', 'summary']), + // Null where the answer was written without the model. + system: z.string().nullable(), + messages: z.array(modelMessageSchema), + tools: z.array(modelToolSchema), +}); + +export type ToolCall = z.infer; +export type InspectedEntry = z.infer; +export type InspectedJob = z.infer; +export type InspectedJobEvent = z.infer; +export type ThreadState = z.infer; +export type InspectedThread = z.infer; +export type Inspection = z.infer; +export type ModelMessage = z.infer; +export type ModelTool = z.infer; +export type ModelRequest = z.infer; + +export function getInspection(org: string, personaId: string) { + return apiRequest('GET', `${orgApi(org)}/chat/personas/${personaId}/inspect`, inspectionSchema); +} + +export function getRequest(org: string, entryId: string) { + return apiRequest('GET', `${orgApi(org)}/chat/inspect/entries/${entryId}/request`, requestSchema); +} diff --git a/packages/web/src/components/inspect-log.tsx b/packages/web/src/components/inspect-log.tsx new file mode 100644 index 00000000..4c55b690 --- /dev/null +++ b/packages/web/src/components/inspect-log.tsx @@ -0,0 +1,196 @@ +import {Badge} from '@proxy/ui/components/badge'; +import {Button} from '@proxy/ui/components/button'; +import {cn} from '@proxy/ui/lib/utils'; +import type {InspectedEntry, InspectedThread, ThreadState} from '@/client/inspect-client'; +import {formatTime} from '@/lib/format'; +import {coveredSeq, prettyJson} from '@/lib/inspect'; + +// The inspector's middle: a thread's whole log as the harness has it, every entry with what the +// model doesn't see (its seq, its cost, its job), and above it where the thread stands for the +// harness: whether the model is due, what holds it, how close the log is to being summarized. + +// Text longer than this is folded, with its length shown. +const FOLD_AT = 600; + +const KIND_CLASS: Record = { + person: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + tool_result: 'bg-muted text-muted-foreground', + event: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', + summary: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', +}; + +function Folded({text, open = false}: {text: string; open?: boolean}) { + if (text.length <= FOLD_AT) { + return 
{text}
; + } + return ( +
+ + {text.length.toLocaleString('en-US')} characters + + 
{text}
+
+ ); +} + +function Calls({entry}: {entry: InspectedEntry}) { + const calls = entry.toolCalls ?? []; + if (calls.length === 0) { + return null; + } + return ( +
    + {calls.map((call) => ( +
  • +
    + {call.name} + {call.id} +
    + 
    +						{prettyJson(call.arguments)}
+
    +
    • + ))} +
+ ); +} + +// What an entry says beside its kind: which call it answers, how far a summary reaches. +function lead(entry: InspectedEntry): string | null { + if (entry.kind === 'tool_result') { + return `answers ${entry.callId ?? '?'}`; + } + if (entry.kind === 'event') { + return `report for ${entry.callId ?? '?'}`; + } + if (entry.kind === 'summary') { + return `covers up to #${entry.coversSeq ?? '?'}`; + } + return null; +} + +function EntryCard({ + entry, + covered, + selected, + onSelect, +}: { + entry: InspectedEntry; + covered: boolean; + selected: boolean; + onSelect: (entryId: string) => void; +}) { + const askable = entry.kind === 'assistant' || entry.kind === 'summary'; + const tokens = entry.inputTokens !== null || entry.outputTokens !== null; + return ( +
  • +
    + #{entry.seq} + + {entry.kind} + + {lead(entry) && {lead(entry)}} + {formatTime(entry.at)} + {tokens && ( + + {(entry.inputTokens ?? 0).toLocaleString('en-US')} in /{' '} + {(entry.outputTokens ?? 0).toLocaleString('en-US')} out + + )} + {entry.pending && pending} + {entry.isError && error} + {covered && summarized} + {entry.jobId !== null && job {entry.jobId.slice(-6)}} + {askable && ( + + )} +
    + {entry.text !== '' && } + +
    • + ); +} + +// Where the thread stands for the harness. +function StateStrip({state, pending}: {state: ThreadState; pending: number}) { + const share = Math.min(1, state.lastInputTokens / state.compactAtTokens); + const holds = + state.pendingCalls.length > 0 + ? `waiting on ${state.pendingCalls.map((call) => call.name).join(', ')}` + : state.ready + ? 'model due' + : 'nothing new for the model'; + return ( +
    + {holds} + {pending > 0 && {pending} pending} + + {state.turnsAlone}/{state.maxTurnsAlone} steps alone + + + + {state.lastInputTokens.toLocaleString('en-US')}/ + {state.compactAtTokens.toLocaleString('en-US')} tokens + + + + + + {state.compactionCoversSeq !== null && ( + next turn summarizes up to #{state.compactionCoversSeq} + )} +
    + ); +} + +export function ThreadLog({ + thread, + selectedEntryId, + onSelectEntry, +}: { + thread: InspectedThread; + selectedEntryId: string | null; + onSelectEntry: (entryId: string) => void; +}) { + const covered = coveredSeq(thread.entries); + const pending = thread.entries.filter((entry) => entry.pending).length; + return ( + <> + +
    +
      + {thread.entries.length === 0 && ( +
    • + Nothing in the log yet. +
      • + )} + {thread.entries.map((entry) => ( + + ))} +
    +
    + + ); +} diff --git a/packages/web/src/components/inspect-request.tsx b/packages/web/src/components/inspect-request.tsx new file mode 100644 index 00000000..a580f038 --- /dev/null +++ b/packages/web/src/components/inspect-request.tsx @@ -0,0 +1,207 @@ +import {useEffect, useState} from 'react'; +import {Badge} from '@proxy/ui/components/badge'; +import {cn} from '@proxy/ui/lib/utils'; +import { + getRequest, + type InspectedEntry, + type InspectedThread, + type ModelMessage, + type ModelRequest, +} from '@/client/inspect-client'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import {formatDuration, prettyJson, timedEvents} from '@/lib/inspect'; + +// The inspector's right column: what the model was sent for the answer picked in the log, as the +// api rebuilds it, the system prompt, the messages and the tools; what came back; and what +// happened to the job that ran it. + +const ROLE_CLASS: Record = { + system: 'bg-muted text-muted-foreground', + user: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + tool: 'bg-muted text-muted-foreground', +}; + +function Section({title, children}: {title: string; children: React.ReactNode}) { + return ( +
    +

    {title}

    + {children} +
    + ); +} + +function MessageCard({message}: {message: ModelMessage}) { + const calls = message.role === 'assistant' ? (message.tool_calls ?? []) : []; + const content = message.role === 'tool' ? message.content : (message.content ?? ''); + return ( +
  • +
    + + {message.role} + + {message.role === 'tool' && {message.tool_call_id}} + {content.length.toLocaleString('en-US')} chars +
    + {content !== '' && ( +
    + + {content.slice(0, 80).replace(/\s+/g, ' ')}… + + 
    {content}
    +
    + )} + {calls.map((call) => ( +
    +
    + {call.function.name} + {call.id} +
    + 
    +						{prettyJson(call.function.arguments)}
+
    +
    + ))} +
    • + ); +} + +// What happened to the job that wrote the entry: each claim, requeue, finish and failure. +function JobHistory({entry, thread}: {entry: InspectedEntry; thread: InspectedThread}) { + const job = thread.jobs.find((candidate) => candidate.id === entry.jobId); + const events = thread.events.filter((event) => event.jobId === entry.jobId); + if (!job) { + return

    No job is kept for this entry.

    ; + } + return ( +
      +
    • + {job.id} · {job.kind} · {job.status} · {job.attempts}{' '} + {job.attempts === 1 ? 'attempt' : 'attempts'} +
      • + {timedEvents(events).map(({event, sinceClaimMs}) => ( +
    • + {formatTime(event.at)} + {event.kind} + {event.workerId !== null && ( + {event.workerId} + )} + {sinceClaimMs !== null && ( + after {formatDuration(sinceClaimMs)} + )} + {event.error !== null && {event.error}} +
      • + ))} +
    + ); +} + +export function RequestPane({ + entry, + thread, +}: { + entry: InspectedEntry | null; + thread: InspectedThread; +}) { + const org = useOrgSlug(); + const [request, setRequest] = useState<{ + entryId: string; + result: ModelRequest | null; + error: string | null; + } | null>(null); + + useEffect(() => { + if (!entry) { + return; + } + let stale = false; + void getRequest(org, entry.id).then((result) => { + if (stale) { + return; + } + setRequest({ + entryId: entry.id, + result: result.isOk() ? result.value : null, + error: result.isErr() ? 'The request could not be read.' : null, + }); + }); + return () => { + stale = true; + }; + }, [org, entry]); + + if (!entry) { + return ( +

    + Pick an answer or a summary in the log to see what the model was sent for it. +

    + ); + } + const loaded = request?.entryId === entry.id ? request : null; + return ( +
    +
    + {loaded === null &&

    Reading…

    } + {loaded?.error !== null && loaded?.error !== undefined && ( +

    {loaded.error}

    + )} + {loaded?.result && loaded.result.system === null && ( +

    + This answer was written without the model, so there was no request; the messages are the + log as it was. +

    + )} + {loaded?.result && ( +

    + {loaded.result.kind === 'summary' ? 'A summarization' : 'A turn'}:{' '} + {loaded.result.messages.length} messages, {loaded.result.tools.length} tools +

    + )} +
    + {loaded?.result && ( + <> +
    +
      + {loaded.result.messages.map((message, index) => ( + + ))} +
    +
    + {loaded.result.tools.length > 0 && ( +
    +
      + {loaded.result.tools.map((tool) => ( +
    • +
      + {tool.name} +

      {tool.description}

      + 
      +												{JSON.stringify(tool.parameters, null, 2)}
+
      +
      +
      • + ))} +
    +
    + )} + + )} +
    +

    + {(entry.inputTokens ?? 0).toLocaleString('en-US')} tokens in,{' '} + {(entry.outputTokens ?? 0).toLocaleString('en-US')} out + {entry.toolCalls && + entry.toolCalls.length > 0 && + `, ${entry.toolCalls.length} ${entry.toolCalls.length === 1 ? 'call' : 'calls'}`} +

    + {entry.text !== '' && ( + 
    {entry.text}
    + )} +
    +
    + +
    +
    + ); +} diff --git a/packages/web/src/components/inspect-threads.tsx b/packages/web/src/components/inspect-threads.tsx new file mode 100644 index 00000000..450885f8 --- /dev/null +++ b/packages/web/src/components/inspect-threads.tsx @@ -0,0 +1,250 @@ +import {Link} from 'react-router'; +import {Badge} from '@proxy/ui/components/badge'; +import {cn} from '@proxy/ui/lib/utils'; +import type {InspectedJob, InspectedThread} from '@/client/inspect-client'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import { + formatDue, + formatDuration, + jobTitle, + lastAttempt, + threadLabel, + threadStatus, + type ThreadStatus, + tokensOf, +} from '@/lib/inspect'; + +// The inspector's left column: the conversation's threads, the conversation first and the +// sub-agents started from it under it, each with where it stands; and under them the queue, +// every job of every thread still to run or running, with its history behind a toggle. + +const STATUS_LABEL: Record = { + working: 'working', + queued: 'queued', + asleep: 'asleep', + waiting: 'waiting', + reported: 'reported', + failed: 'failed', +}; + +const STATUS_CLASS: Record = { + working: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + queued: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', + asleep: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', + waiting: 'bg-muted text-muted-foreground', + reported: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + failed: 'bg-destructive/10 text-destructive', +}; + +export function StatusBadge({status}: {status: ThreadStatus}) { + return ( + + {STATUS_LABEL[status.kind]} + {status.until !== null && ` until ${formatTime(status.until)}`} + + ); +} + +function ThreadRow({ + thread, + now, + current, + onSelect, +}: { + thread: InspectedThread; + now: string; + current: boolean; + onSelect: (threadId: string) => void; +}) { + const org = useOrgSlug(); + const tokens = tokensOf(thread.entries); + const sub = thread.parentThreadId !== null; + return ( +
  • + + {thread.taskId !== null && ( + + Task in the activity log + + )} +
    • + ); +} + +export function ThreadTree({ + threads, + now, + current, + onSelect, +}: { + threads: InspectedThread[]; + now: string; + current: string; + onSelect: (threadId: string) => void; +}) { + return ( +
    +

    + Threads +

    +
      + {threads.map((thread) => ( + + ))} +
    +
    + ); +} + +const JOB_CLASS: Record = { + running: STATUS_CLASS.working, + queued: STATUS_CLASS.queued, + done: STATUS_CLASS.reported, + failed: STATUS_CLASS.failed, +}; + +// Running first, then queued by when they are due, then what is over, newest first. +function order(a: InspectedJob, b: InspectedJob): number { + const rank = (job: InspectedJob) => + job.status === 'running' ? 0 : job.status === 'queued' ? 1 : 2; + if (rank(a) !== rank(b)) { + return rank(a) - rank(b); + } + if (a.status === 'queued') { + return a.runAt.localeCompare(b.runAt); + } + return b.createdAt.localeCompare(a.createdAt); +} + +// What a job is up to, in a line: due when, held by whom until when, or how long it ran. +function jobDetail(job: InspectedJob, thread: InspectedThread, now: string): string { + if (job.status === 'queued') { + return `${formatDue(job.runAt, now)}${job.attempts > 0 ? `, attempt ${job.attempts + 1}` : ''}`; + } + const attempt = lastAttempt(job, thread.events); + if (job.status === 'running') { + const lease = + job.leaseExpiresAt === null + ? '' + : `, lease ${formatDue(job.leaseExpiresAt, now).replace('in ', '')} left`; + const since = + attempt.claimedAt === null + ? '' + : `, ${formatDuration(new Date(now).getTime() - new Date(attempt.claimedAt).getTime())} so far`; + return `${job.claimedBy ?? 'claimed'}${lease}${since}, attempt ${job.attempts}`; + } + const ran = attempt.ranMs === null ? '' : `ran ${formatDuration(attempt.ranMs)}, `; + return `${ran}${job.attempts} ${job.attempts === 1 ? 'attempt' : 'attempts'}`; +} + +function JobRow({ + job, + thread, + now, + onSelectThread, +}: { + job: InspectedJob; + thread: InspectedThread; + now: string; + onSelectThread: (threadId: string) => void; +}) { + return ( +
  • + + + {job.status} + + {jobTitle(job, thread.entries)} + + + {jobDetail(job, thread, now)} + {job.error !== null && {job.error}} +
    • + ); +} + +export function JobQueue({ + threads, + now, + showHistory, + onToggleHistory, + onSelectThread, +}: { + threads: InspectedThread[]; + now: string; + showHistory: boolean; + onToggleHistory: () => void; + onSelectThread: (threadId: string) => void; +}) { + const jobs = threads + .flatMap((thread) => thread.jobs.map((job) => ({job, thread}))) + .filter(({job}) => showHistory || job.status === 'queued' || job.status === 'running') + .sort((a, b) => order(a.job, b.job)); + return ( +
    +

    + Jobs + +

    + {jobs.length === 0 && ( +

    Nothing queued or running.

    + )} +
      + {jobs.map(({job, thread}) => ( + + ))} +
    +
    + ); +} diff --git a/packages/web/src/lib/inspect.test.ts b/packages/web/src/lib/inspect.test.ts new file mode 100644 index 00000000..3ef630db --- /dev/null +++ b/packages/web/src/lib/inspect.test.ts @@ -0,0 +1,187 @@ +import {describe, expect, test} from 'bun:test'; +import type {InspectedEntry, InspectedJob, InspectedThread} from '@/client/inspect-client'; +import { + coveredSeq, + formatDue, + formatDuration, + jobTitle, + lastAttempt, + threadStatus, + tokensOf, +} from './inspect'; + +const now = '2026-10-21T10:00:00.000Z'; + +function job(extra: Partial): InspectedJob { + return { + id: 'job-1', + threadId: 'thread-1', + kind: 'inference', + status: 'queued', + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 0, + error: null, + createdAt: now, + ...extra, + }; +} + +function entry(extra: Partial): InspectedEntry { + return { + id: 'e1', + seq: 1, + kind: 'person', + text: '', + toolCalls: null, + callId: null, + isError: false, + pending: false, + inputTokens: null, + outputTokens: null, + coversSeq: null, + hasSystem: false, + jobId: null, + at: now, + ...extra, + }; +} + +function thread(extra: Partial): InspectedThread { + return { + id: 'thread-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, + entries: [], + jobs: [], + events: [], + state: { + ready: false, + pendingCalls: [], + turnsAlone: 0, + maxTurnsAlone: 30, + lastInputTokens: 0, + compactAtTokens: 80_000, + compactionCoversSeq: null, + }, + ...extra, + }; +} + +describe('threadStatus', () => { + test('is at work while a job runs, and queued while one is due', () => { + expect(threadStatus(thread({jobs: [job({status: 'running'})]}), now).kind).toBe('working'); + expect(threadStatus(thread({jobs: [job({status: 'queued'})]}), now).kind).toBe('queued'); + }); + + test('is asleep until the first of its jobs due later, and only when every open job is', () => { + const later = job({id: 'j2', kind: 'tool', runAt: '2026-10-21T10:05:00.000Z'}); + const sooner = job({id: 'j3', kind: 'tool', runAt: '2026-10-21T10:01:00.000Z'}); + expect(threadStatus(thread({jobs: [later, sooner]}), now)).toEqual({ + kind: 'asleep', + until: '2026-10-21T10:01:00.000Z', + }); + expect(threadStatus(thread({jobs: [later, job({status: 'queued'})]}), now).kind).toBe('queued'); + }); + + test('waits for the person with nothing open, and a finished sub-agent reported, or failed where a job did', () => { + expect(threadStatus(thread({jobs: [job({status: 'done'})]}), now).kind).toBe('waiting'); + expect(threadStatus(thread({finishedAt: now, jobs: [job({status: 'done'})]}), now).kind).toBe( + 'reported', + ); + expect(threadStatus(thread({finishedAt: now, jobs: [job({status: 'failed'})]}), now).kind).toBe( + 'failed', + ); + }); +}); + +describe('jobTitle', () => { + test('names a tool job by its call, found in the entry it is in', () => { + const entries = [ + entry({id: 'e2', kind: 'assistant', toolCalls: [{id: 'c1', name: 'read', arguments: '{}'}]}), + ]; + expect(jobTitle(job({kind: 'tool', entryId: 'e2', callId: 'c1'}), entries)).toBe('read'); + expect(jobTitle(job({kind: 'tool', entryId: 'e9', callId: 'c1'}), entries)).toBe('tool'); + expect(jobTitle(job({}), entries)).toBe('inference'); + }); +}); + +describe('the log’s sums', () => { + test('add up what the answers cost, and say how far the latest summary reaches', () => { + const entries = [ + entry({id: 'e1', kind: 'assistant', inputTokens: 100, outputTokens: 10}), + entry({id: 'e2', seq: 2, kind: 'summary', coversSeq: 1, inputTokens: 50, outputTokens: 5}), + entry({id: 'e3', seq: 3, kind: 'assistant', inputTokens: 20, outputTokens: 2}), + ]; + expect(tokensOf(entries)).toEqual({input: 170, output: 17}); + expect(coveredSeq(entries)).toBe(1); + expect(coveredSeq([])).toBe(-1); + }); +}); + +describe('the times', () => { + test('are formatted to what matters at their size', () => { + expect(formatDuration(340)).toBe('340ms'); + expect(formatDuration(1234)).toBe('1.2s'); + expect(formatDuration(130_000)).toBe('2m 10s'); + expect(formatDue('2026-10-21T10:00:12.000Z', now)).toBe('in 12.0s'); + expect(formatDue('2026-10-21T09:00:00.000Z', now)).toBe('due'); + }); + + test('give a job’s last attempt from its events: the claim, and how long it ran once it ended', () => { + const events = [ + { + id: 'ev1', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed' as const, + workerId: 'w1', + error: null, + at: '2026-10-21T10:00:00.000Z', + }, + { + id: 'ev2', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'requeued' as const, + workerId: 'w1', + error: 'down', + at: '2026-10-21T10:00:01.000Z', + }, + { + id: 'ev3', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed' as const, + workerId: 'w2', + error: null, + at: '2026-10-21T10:00:06.000Z', + }, + ]; + expect(lastAttempt(job({}), events)).toEqual({ + claimedAt: '2026-10-21T10:00:06.000Z', + ranMs: null, + }); + const done = { + id: 'ev4', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'done' as const, + workerId: 'w2', + error: null, + at: '2026-10-21T10:00:08.500Z', + }; + expect(lastAttempt(job({}), [...events, done])).toEqual({ + claimedAt: '2026-10-21T10:00:06.000Z', + ranMs: 2500, + }); + expect(lastAttempt(job({}), [])).toEqual({claimedAt: null, ranMs: null}); + }); +}); diff --git a/packages/web/src/lib/inspect.ts b/packages/web/src/lib/inspect.ts new file mode 100644 index 00000000..213fd873 --- /dev/null +++ b/packages/web/src/lib/inspect.ts @@ -0,0 +1,144 @@ +import type { + InspectedEntry, + InspectedJob, + InspectedJobEvent, + InspectedThread, +} from '@/client/inspect-client'; + +/** + * What the inspector says of a thread, a job and a log, worked out from what the api gives: + * where a thread stands, what a job is for and how long it took, what a summary covers. + */ + +/** Where a thread stands, in a word: at work, queued or asleep on a job, waiting for the person (a conversation), reported or failed (a sub-agent). */ +export type ThreadStatus = { + kind: 'working' | 'queued' | 'asleep' | 'waiting' | 'reported' | 'failed'; + // Until when, for one asleep. + until: string | null; +}; + +const SLEEP_SOON_MS = 1000; + +export function threadStatus(thread: InspectedThread, now: string): ThreadStatus { + if (thread.finishedAt !== null) { + const failed = thread.jobs.some((job) => job.status === 'failed'); + return {kind: failed ? 'failed' : 'reported', until: null}; + } + const open = thread.jobs.filter((job) => job.status === 'queued' || job.status === 'running'); + if (open.some((job) => job.status === 'running')) { + return {kind: 'working', until: null}; + } + if (open.length === 0) { + return {kind: 'waiting', until: null}; + } + const time = new Date(now).getTime(); + // A queued job due later is a sleep; one due now is about to be claimed. + const later = open.filter((job) => new Date(job.runAt).getTime() - time > SLEEP_SOON_MS); + if (later.length === open.length) { + const first = [...later].sort((a, b) => a.runAt.localeCompare(b.runAt))[0]; + return {kind: 'asleep', until: first?.runAt ?? null}; + } + return {kind: 'queued', until: null}; +} + +/** What a thread is called in a list: the conversation, or a sub-agent by its title. */ +export function threadLabel(thread: InspectedThread): string { + return thread.parentThreadId === null ? 'Conversation' : (thread.title ?? 'Sub-agent'); +} + +/** Each of a job's events with how long after the attempt's claim it came; null for a claim, and before any. */ +export function timedEvents( + events: InspectedJobEvent[], +): Array<{event: InspectedJobEvent; sinceClaimMs: number | null}> { + let claimedAt: string | null = null; + return events.map((event) => { + const since = + event.kind !== 'claimed' && claimedAt !== null + ? new Date(event.at).getTime() - new Date(claimedAt).getTime() + : null; + if (event.kind === 'claimed') { + claimedAt = event.at; + } + return {event, sinceClaimMs: since}; + }); +} + +/** What a job is for: the model's turn, or the call it runs, by name, found in the entry the call is in. */ +export function jobTitle(job: InspectedJob, entries: InspectedEntry[]): string { + if (job.kind === 'inference') { + return 'inference'; + } + const entry = entries.find((candidate) => candidate.id === job.entryId); + const call = entry?.toolCalls?.find((candidate) => candidate.id === job.callId); + return call ? call.name : 'tool'; +} + +/** The sum of what the model's answers in the log cost. */ +export function tokensOf(entries: InspectedEntry[]): {input: number; output: number} { + return entries.reduce( + (sum, entry) => ({ + input: sum.input + (entry.inputTokens ?? 0), + output: sum.output + (entry.outputTokens ?? 0), + }), + {input: 0, output: 0}, + ); +} + +/** The seq of the last entry the log's latest summary stands for; -1 where there is none. */ +export function coveredSeq(entries: InspectedEntry[]): number { + return entries.findLast((entry) => entry.kind === 'summary')?.coversSeq ?? -1; +} + +/** "340ms", "1.2s", "2m 10s". */ +export function formatDuration(ms: number): string { + if (ms < 1000) { + return `${Math.max(0, Math.round(ms))}ms`; + } + if (ms < 60_000) { + return `${(ms / 1000).toFixed(1)}s`; + } + const minutes = Math.floor(ms / 60_000); + const seconds = Math.round((ms % 60_000) / 1000); + return `${minutes}m ${seconds}s`; +} + +/** "in 12s", "in 3m 0s"; "due" once the time has come. */ +export function formatDue(runAt: string, now: string): string { + const ms = new Date(runAt).getTime() - new Date(now).getTime(); + if (ms <= 0) { + return 'due'; + } + return `in ${formatDuration(ms)}`; +} + +/** The job's last attempt, from its events: when it was claimed, and how long it ran, where it ended. */ +export function lastAttempt( + job: InspectedJob, + events: InspectedJobEvent[], +): {claimedAt: string | null; ranMs: number | null} { + const own = events.filter((event) => event.jobId === job.id); + const claimed = own.findLast((event) => event.kind === 'claimed') ?? null; + const ended = + own.findLast( + (event) => event.kind !== 'claimed' && (claimed === null || event.at >= claimed.at), + ) ?? null; + if (!claimed) { + return {claimedAt: null, ranMs: null}; + } + if (!ended) { + return {claimedAt: claimed.at, ranMs: null}; + } + return { + claimedAt: claimed.at, + ranMs: new Date(ended.at).getTime() - new Date(claimed.at).getTime(), + }; +} + +/** The arguments pretty-printed where they are JSON, as written otherwise. */ +export function prettyJson(text: string): string { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } +} diff --git a/packages/web/src/loaders.ts b/packages/web/src/loaders.ts index f488e1c4..79ebd38a 100644 --- a/packages/web/src/loaders.ts +++ b/packages/web/src/loaders.ts @@ -10,6 +10,7 @@ import { } from '@/client/agents-client'; import {getConnection, listConnections} from '@/client/connections-client'; import {getThread, listPersonas} from '@/client/harness-client'; +import {getInspection} from '@/client/inspect-client'; import {getMe} from '@/client/me-client'; import {getAuthorizationRequest} from '@/client/oauth-client'; import {getInvitation, getOrg, listMembers, listOrgs} from '@/client/orgs-client'; @@ -132,6 +133,30 @@ export async function chatLoader(args: OrgParams & {params: {id?: string}}) { return {log: unwrapLoaderResult(log), personas: unwrapLoaderResult(personas).personas}; } +// The inspector of the conversation with a persona: the first persona's where the address names none. +export async function inspectIndexLoader(args: OrgParams) { + const org = orgOf(args); + const {personas} = unwrapLoaderResult(await listPersonas(org)); + const first = personas[0]; + if (!first) { + throw redirect(`/${org}`); + } + throw redirect(`/${org}/inspect/${first.id}`); +} + +// The conversation with a persona as the harness has it, and the personas to switch between. +export async function inspectLoader(args: OrgParams & {params: {id?: string}}) { + const org = orgOf(args); + const [inspection, personas] = await Promise.all([ + getInspection(org, args.params.id ?? ''), + listPersonas(org), + ]); + return { + inspection: unwrapLoaderResult(inspection), + personas: unwrapLoaderResult(personas).personas, + }; +} + // The vault is the one built-in connection, which every organization has twice: the person's own // there and the organization's, shared with everyone in it. The entries of each are kept in // Personal Agent Proxy. diff --git a/packages/web/src/pages/inspect-page.test.tsx b/packages/web/src/pages/inspect-page.test.tsx new file mode 100644 index 00000000..4d7b8f51 --- /dev/null +++ b/packages/web/src/pages/inspect-page.test.tsx @@ -0,0 +1,329 @@ +/// + +import {afterEach, beforeEach, describe, expect, mock, test} from 'bun:test'; +import {cleanup, fireEvent, render, screen, waitFor, within} from '@testing-library/react'; +import {createMemoryRouter, RouterProvider} from 'react-router'; +import {TooltipProvider} from '@proxy/ui/components/tooltip'; +import {ThemeProvider} from '@proxy/ui/components/theme-provider'; +import {HumanSide} from '@/components/human-side'; +import type {ChatPersona} from '@/client/harness-client'; +import type { + InspectedEntry, + InspectedJob, + InspectedThread, + Inspection, + ModelRequest, +} from '@/client/inspect-client'; +import {InspectPage} from '@/pages/inspect-page'; +import {testHuman} from '@/test-org'; + +const now = '2026-10-21T10:00:00.000Z'; +const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; +const persona: ChatPersona = { + id: 'persona-1', + name: 'Finance', + look, + color: 'blue', + mayChange: true, + lastMessage: null, +}; + +function entry( + seq: number, + kind: InspectedEntry['kind'], + text: string, + extra: Partial = {}, +): InspectedEntry { + return { + id: `e${seq}`, + seq, + kind, + text, + toolCalls: null, + callId: null, + isError: false, + pending: false, + inputTokens: null, + outputTokens: null, + coversSeq: null, + hasSystem: false, + jobId: null, + at: now, + ...extra, + }; +} + +function job(id: string, threadId: string, extra: Partial = {}): InspectedJob { + return { + id, + threadId, + kind: 'inference', + status: 'done', + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 1, + error: null, + createdAt: now, + ...extra, + }; +} + +const state = { + ready: false, + pendingCalls: [], + turnsAlone: 1, + maxTurnsAlone: 30, + lastInputTokens: 1200, + compactAtTokens: 80_000, + compactionCoversSeq: null, +}; + +// The person asked for invoices, the agent delegated, and the sub-agent is reading. +const conversation: InspectedThread = { + id: 'thread-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, + entries: [ + entry(1, 'person', 'Find my invoices'), + entry(2, 'assistant', '', { + toolCalls: [ + { + id: 'c1', + name: 'delegate', + arguments: '{"title":"Find the invoices","brief":"Find them"}', + }, + ], + inputTokens: 1200, + outputTokens: 40, + hasSystem: true, + jobId: 'job-1', + }), + entry(3, 'tool_result', 'Started the sub-agent “Find the invoices”.', { + callId: 'c1', + jobId: 'job-2', + }), + entry(9, 'person', 'Thanks', {pending: true}), + ], + jobs: [ + job('job-1', 'thread-1'), + job('job-2', 'thread-1', {kind: 'tool', entryId: 'e2', callId: 'c1'}), + ], + events: [ + { + id: 'ev1', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:58.000Z', + }, + { + id: 'ev2', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'done', + workerId: 'host-1', + error: null, + at: '2026-10-21T10:00:00.500Z', + }, + ], + state, +}; +const subAgent: InspectedThread = { + id: 'thread-2', + parentThreadId: 'thread-1', + parentCallId: 'c1', + title: 'Find the invoices', + taskId: 'task-2', + finishedAt: null, + createdAt: now, + entries: [ + entry(4, 'person', 'Find them'), + entry(5, 'assistant', '', { + toolCalls: [ + {id: 'c2', name: 'read', arguments: '{"connection":"gmail-1","tool":"list_emails"}'}, + ], + inputTokens: 800, + outputTokens: 30, + hasSystem: true, + jobId: 'job-3', + }), + ], + jobs: [ + job('job-3', 'thread-2'), + job('job-4', 'thread-2', { + kind: 'tool', + status: 'running', + entryId: 'e5', + callId: 'c2', + claimedBy: 'host-1', + leaseExpiresAt: '2026-10-21T10:02:00.000Z', + }), + ], + events: [ + { + id: 'ev4', + jobId: 'job-3', + threadId: 'thread-2', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:50.000Z', + }, + { + id: 'ev5', + jobId: 'job-3', + threadId: 'thread-2', + kind: 'done', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:53.000Z', + }, + { + id: 'ev3', + jobId: 'job-4', + threadId: 'thread-2', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:59.000Z', + }, + ], + state: { + ...state, + pendingCalls: [{id: 'c2', name: 'read', arguments: '{}'}], + lastInputTokens: 800, + }, +}; + +const inspection: Inspection = {persona, now, threads: [conversation, subAgent]}; + +const request: ModelRequest = { + kind: 'turn', + system: 'You are signed in.', + messages: [ + {role: 'system', content: 'You are signed in.'}, + {role: 'user', content: 'Find my invoices'}, + ], + tools: [{name: 'delegate', description: 'Delegates', parameters: {type: 'object'}}], +}; + +const realFetch = globalThis.fetch; +const fetchMock = mock<(input: string, init?: RequestInit) => Promise>(); + +beforeEach(() => { + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => { + if (input.endsWith('/request')) { + return Promise.resolve(Response.json(request)); + } + if (input.endsWith('/inspect')) { + return Promise.resolve(Response.json(inspection)); + } + return Promise.resolve(Response.json({error: 'unexpected'}, {status: 500})); + }); + globalThis.fetch = fetchMock as unknown as typeof fetch; +}); + +afterEach(() => { + cleanup(); + globalThis.fetch = realFetch; +}); + +function renderInspectPage() { + const router = createMemoryRouter( + [ + { + id: 'human', + loader: () => testHuman(), + element: , + children: [ + { + path: '/:org/inspect/:id', + loader: () => ({inspection, personas: [persona]}), + element: , + }, + ], + }, + ], + {initialEntries: ['/acme/inspect/persona-1']}, + ); + return render( + + + + + , + ); +} + +describe('InspectPage', () => { + test('lists the conversation and its sub-agents with where each stands, and the jobs open on them', async () => { + renderInspectPage(); + const threads = await screen.findByRole('region', {name: 'Threads'}); + const rows = within(threads).getAllByRole('button'); + expect(rows[0]?.textContent).toContain('Conversation'); + expect(rows[0]?.textContent).toContain('waiting'); + expect(rows[1]?.textContent).toContain('Find the invoices'); + expect(rows[1]?.textContent).toContain('working'); + expect(rows[1]?.textContent).toContain('1 open calls'); + expect( + within(threads).getByRole('link', {name: 'Task in the activity log'}).getAttribute('href'), + ).toBe('/acme/activity/tasks/task-2'); + + const jobs = screen.getByRole('region', {name: 'Jobs'}); + expect(jobs.textContent).toContain('running'); + expect(jobs.textContent).toContain('read'); + expect(jobs.textContent).toContain('host-1'); + expect(jobs.textContent).not.toContain('inference'); + fireEvent.click(within(jobs).getByRole('button', {name: 'With history'})); + expect(jobs.textContent).toContain('inference'); + expect(jobs.textContent).toContain('ran 2.5s'); + }); + + test('shows the conversation’s whole log, the calls with their arguments and what is pending', async () => { + renderInspectPage(); + const log = await screen.findByRole('list', {name: 'Log'}); + // The log's own items, not the calls listed within one. + const items = within(log) + .getAllByRole('listitem') + .filter((item) => item.parentElement === log); + expect(items.map((item) => item.textContent)).toEqual([ + expect.stringContaining('#1person'), + expect.stringContaining('delegate'), + expect.stringContaining('answers c1'), + expect.stringContaining('pending'), + ]); + expect(items[1]?.textContent).toContain('"title": "Find the invoices"'); + expect(items[1]?.textContent).toContain('1,200 in / 40 out'); + expect(screen.getByText(/nothing new for the model/)).toBeDefined(); + expect(screen.getByText('1 pending')).toBeDefined(); + }); + + test('opens a sub-agent’s log, and the request behind an answer in it', async () => { + renderInspectPage(); + const threads = await screen.findByRole('region', {name: 'Threads'}); + fireEvent.click(within(threads).getByRole('button', {name: /Find the invoices/})); + expect(screen.getByText(/waiting on read/)).toBeDefined(); + const log = screen.getByRole('list', {name: 'Log'}); + expect(log.textContent).toContain('list_emails'); + + fireEvent.click(within(log).getByRole('button', {name: 'Request'})); + await waitFor(() => { + expect(screen.getByText(/A turn: 2 messages, 1 tools/)).toBeDefined(); + }); + expect(fetchMock.mock.calls[0]?.[0]).toBe('/api/orgs/acme/chat/inspect/entries/e5/request'); + expect(screen.getByText('Find my invoices', {selector: 'pre'})).toBeDefined(); + expect(screen.getByText('delegate', {selector: 'summary'})).toBeDefined(); + expect(screen.getByText('claimed')).toBeDefined(); + expect(screen.getByText('after 3.0s')).toBeDefined(); + }); +}); diff --git a/packages/web/src/pages/inspect-page.tsx b/packages/web/src/pages/inspect-page.tsx new file mode 100644 index 00000000..c114a09d --- /dev/null +++ b/packages/web/src/pages/inspect-page.tsx @@ -0,0 +1,217 @@ +import {ArrowUpIcon} from 'lucide-react'; +import {type FormEvent, type KeyboardEvent, useCallback, useEffect, useRef, useState} from 'react'; +import {Link, useLoaderData, useNavigate} from 'react-router'; +import {Button} from '@proxy/ui/components/button'; +import { + InputGroup, + InputGroupAddon, + InputGroupButton, + InputGroupTextarea, +} from '@proxy/ui/components/input-group'; +import {postMessage} from '@/client/harness-client'; +import {getInspection, type Inspection} from '@/client/inspect-client'; +import {Head} from '@/components/head'; +import {ThreadLog} from '@/components/inspect-log'; +import {RequestPane} from '@/components/inspect-request'; +import {JobQueue, StatusBadge, ThreadTree} from '@/components/inspect-threads'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import {threadLabel, threadStatus} from '@/lib/inspect'; +import {describeFetchError} from '@/lib/loader-utils'; +import type {inspectLoader} from '@/loaders'; + +/** + * The inspector: the person's conversation with a persona as the harness has it, on a page of + * its own, for whoever works on the built-in agent. Nothing of the Chat page: the conversation's + * threads and the queue on the left, the thread picked in the middle with its whole log and an + * input to say something in the conversation, and on the right what the model was sent for the + * answer picked. Read again as the agent works, as the Chat page is. + */ + +const BUSY_POLL_MS = 1000; +const IDLE_POLL_MS = 4000; + +type Loaded = Awaited>; + +export function InspectPage() { + const loaded = useLoaderData(); + return ; +} + +function hasOpenJobs(inspection: Inspection): boolean { + return inspection.threads.some((thread) => + thread.jobs.some((job) => job.status === 'queued' || job.status === 'running'), + ); +} + +function Inspector({loaded}: {loaded: Loaded}) { + const org = useOrgSlug(); + const navigate = useNavigate(); + const [inspection, setInspection] = useState(loaded.inspection); + const [threadId, setThreadId] = useState(loaded.inspection.threads[0]?.id ?? ''); + const [entryId, setEntryId] = useState(null); + const [showHistory, setShowHistory] = useState(false); + const [text, setText] = useState(''); + const [sending, setSending] = useState(false); + const [error, setError] = useState(null); + const form = useRef(null); + const personaId = loaded.inspection.persona.id; + const busy = hasOpenJobs(inspection); + + const latestRead = useRef(0); + const read = useCallback(async () => { + latestRead.current += 1; + const own = latestRead.current; + const result = await getInspection(org, personaId); + if (own !== latestRead.current || result.isErr()) { + return; + } + setInspection(result.value); + }, [org, personaId]); + + useEffect(() => { + const timer = setInterval(() => void read(), busy ? BUSY_POLL_MS : IDLE_POLL_MS); + return () => { + clearInterval(timer); + }; + }, [read, busy]); + + const thread = + inspection.threads.find((candidate) => candidate.id === threadId) ?? inspection.threads[0]; + const entry = thread?.entries.find((candidate) => candidate.id === entryId) ?? null; + + const send = async (event: FormEvent) => { + event.preventDefault(); + const message = text.trim(); + if (message === '' || sending) { + return; + } + setSending(true); + setError(null); + const posted = await postMessage(org, personaId, message); + setSending(false); + if (posted.isErr()) { + setError(describeFetchError(posted.error)); + return; + } + setText(''); + await read(); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Enter' && !event.shiftKey) { + event.preventDefault(); + form.current?.requestSubmit(); + } + }; + + const persona = inspection.persona; + return ( +
    +
    + + Inspector + + + Open in Chat + + + + {busy ? 'live, every second' : 'idle, every 4 seconds'} + + {formatTime(inspection.now)} + +
    +
    + +
    + {thread && ( + <> +
    + {threadLabel(thread)} + + {thread.id} + {thread.parentCallId !== null && ( + + from {thread.parentCallId} + + )} +
    + + {thread.parentThreadId === null && ( +
    void send(event)} + className="flex shrink-0 flex-col gap-1 border-t p-3" + > + {error &&

    {error}

    } + + setText(event.target.value)} + onKeyDown={onKeyDown} + placeholder={`Message ${persona.name}`} + aria-label="Message" + className="min-h-9 p-2 text-sm" + rows={1} + /> + + + + + + +
    + )} + + )} +
    + +
    +
    + ); +} diff --git a/packages/web/src/router.tsx b/packages/web/src/router.tsx index 78f3b5ac..34df3e71 100644 --- a/packages/web/src/router.tsx +++ b/packages/web/src/router.tsx @@ -27,6 +27,8 @@ import { connectionLoader, connectionsLoader, homeLoader, + inspectIndexLoader, + inspectLoader, inviteLoader, membersLoader, shouldRevalidateHome, @@ -59,6 +61,7 @@ import {ConnectAgentPage} from '@/pages/connect-agent-page'; import {ConnectionPage} from '@/pages/connection-page'; import {ConnectionsPage} from '@/pages/connections-page'; import {HomePage} from '@/pages/home-page'; +import {InspectPage} from '@/pages/inspect-page'; import {InvitePage} from '@/pages/invite-page'; import {LoginPage} from '@/pages/login-page'; import {MembersPage} from '@/pages/members-page'; @@ -269,6 +272,10 @@ export const router = createBrowserRouter([ // The built-in agents, behind the `harness` experiment: the api refuses it otherwise. {path: '/:org/chat', loader: chatIndexLoader, element: }, {path: '/:org/chat/:id', loader: chatLoader, element: }, + // The inspector of the built-in agent, behind the `harness_debug` experiment too, on a + // page of its own, reached by its address alone. + {path: '/:org/inspect', loader: inspectIndexLoader, element: }, + {path: '/:org/inspect/:id', loader: inspectLoader, element: }, {path: '/:org/agents', loader: agentsLoader, element: }, {path: '/:org/agents/new', loader: agentsLoader, element: }, {path: '/:org/agents/:id', loader: agentLoader, element: }, From 626552974b905e8bda04dfed4fb7c574760c10c7 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:01:14 +0000 Subject: [PATCH 2/5] call the inspector's experiment debug --- packages/api/src/harness/inspect-route.test.ts | 2 +- packages/api/src/harness/inspect-route.ts | 4 ++-- packages/api/src/server/create-api-router.ts | 2 +- packages/utils/src/experiments.ts | 6 +++--- packages/web/src/router.tsx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/api/src/harness/inspect-route.test.ts b/packages/api/src/harness/inspect-route.test.ts index 2d479876..a2aa8eba 100644 --- a/packages/api/src/harness/inspect-route.test.ts +++ b/packages/api/src/harness/inspect-route.test.ts @@ -39,7 +39,7 @@ mock.module('@proxy/db/harness', () => ({ requeueJob: mock(), })); -const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'harness_debug']}; +const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'debug']}; const requireViewer = mock(async () => Ok(viewer)); mock.module('../utils/user-org', () => ({requireViewer})); diff --git a/packages/api/src/harness/inspect-route.ts b/packages/api/src/harness/inspect-route.ts index 3da15cc1..618c4738 100644 --- a/packages/api/src/harness/inspect-route.ts +++ b/packages/api/src/harness/inspect-route.ts @@ -27,7 +27,7 @@ import {type PersonaResponse, toPersona} from './persona-response'; import {toolsFor} from './tool-surface'; /** - * The inspector's routes, for whoever works on the built-in agent, behind the `harness_debug` + * The inspector's routes, for whoever works on the built-in agent, behind the `debug` * experiment beside `harness`: the viewer's conversation with a persona as the harness has it, * every thread of it with its whole log, its jobs and what happened to each, and, for any answer * of the model, the request it was sent, rebuilt from the log with the functions the harness runs @@ -164,7 +164,7 @@ function stateOf(entries: EntryRow[]): ThreadState { function requireInspector(request: AuthenticatedRequest): Promise> { return Do(async ($) => { const viewer = $(await requireViewer(request)); - if (!hasExperiment(viewer, 'harness') || !hasExperiment(viewer, 'harness_debug')) { + if (!hasExperiment(viewer, 'harness') || !hasExperiment(viewer, 'debug')) { return $(Err(ApiErr.forbidden())); } return viewer; diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index 731c5493..c4707b73 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -220,7 +220,7 @@ export function createApiRouter(): express.Router { org.delete('/chat/personas/:personaId', withAuthResult('Delete persona route', handleDeletePersonaRoute)); org.get('/chat/personas/:personaId/thread', withAuthResult('Get chat thread route', handleGetThreadRoute)); org.post('/chat/personas/:personaId/messages', withAuthResult('Post chat message route', handlePostMessageRoute)); - // The inspector, behind the `harness_debug` experiment too: the conversation as the harness has + // The inspector, behind the `debug` experiment too: the conversation as the harness has // it, and what the model was sent for an answer. org.get('/chat/personas/:personaId/inspect', withAuthResult('Inspect chat thread route', handleInspectThreadRoute)); org.get('/chat/inspect/entries/:entryId/request', withAuthResult('Inspect chat request route', handleInspectRequestRoute)); diff --git a/packages/utils/src/experiments.ts b/packages/utils/src/experiments.ts index bc4b5ab2..11062aec 100644 --- a/packages/utils/src/experiments.ts +++ b/packages/utils/src/experiments.ts @@ -11,9 +11,9 @@ export const EXPERIMENTS = { // The built-in agent: a conversation with an agent Personal Agent Proxy runs itself, behind its // own gate, on the Chat page. harness: 'Built-in agent', - // The inspector of the built-in agent, for whoever works on it: every thread of a conversation - // with its whole log, its jobs and what the model was sent, on its own page. - harness_debug: 'Built-in agent inspector', + // Debugging: the inspector of the built-in agent, for whoever works on it, every thread of a + // conversation with its whole log, its jobs and what the model was sent, on its own page. + debug: 'Debugging', } as const satisfies Record; export type Experiment = keyof typeof EXPERIMENTS; diff --git a/packages/web/src/router.tsx b/packages/web/src/router.tsx index 34df3e71..c0148ee2 100644 --- a/packages/web/src/router.tsx +++ b/packages/web/src/router.tsx @@ -272,7 +272,7 @@ export const router = createBrowserRouter([ // The built-in agents, behind the `harness` experiment: the api refuses it otherwise. {path: '/:org/chat', loader: chatIndexLoader, element: }, {path: '/:org/chat/:id', loader: chatLoader, element: }, - // The inspector of the built-in agent, behind the `harness_debug` experiment too, on a + // The inspector of the built-in agent, behind the `debug` experiment too, on a // page of its own, reached by its address alone. {path: '/:org/inspect', loader: inspectIndexLoader, element: }, {path: '/:org/inspect/:id', loader: inspectLoader, element: }, From 11c63fc3cd3b7eeee6024e3c927244aa8b14d0ec Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:34:57 +0000 Subject: [PATCH 3/5] begin the conversation from the inspector too, where the person has none yet --- packages/api/src/harness/inspect-route.test.ts | 11 +++++++++-- packages/api/src/harness/inspect-route.ts | 8 ++++---- packages/api/src/harness/route.ts | 5 ++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/api/src/harness/inspect-route.test.ts b/packages/api/src/harness/inspect-route.test.ts index 971486d4..714b008b 100644 --- a/packages/api/src/harness/inspect-route.test.ts +++ b/packages/api/src/harness/inspect-route.test.ts @@ -19,6 +19,9 @@ const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', ro const requireViewer = mock(async () => Ok(viewer)); mock.module('../utils/user-org', () => ({requireViewer})); +const ensureBuiltInAgent = mock(async () => Ok({id: 'login-1'})); +mock.module('./built-in-agent', () => ({ensureBuiltInAgent})); + const now = new Date('2026-10-21T10:00:00Z'); const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; const persona = {id: 'persona-1', name: 'Finance', look, color: 'blue', createdById: 'user-1'}; @@ -174,9 +177,13 @@ describe('the inspection of a conversation', () => { expect(agent?.state).toMatchObject({ready: false, pendingCalls: [{id: 'c2', name: 'read'}], turnsAlone: 1, lastInputTokens: 800}); }); - test('is not found before the viewer has a conversation with the persona', async () => { + test('begins the conversation where the viewer has none yet, as the Chat page does', async () => { store.getMemberThread.mockResolvedValue(Ok(null)); - expect((await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('not_found'); + store.ensureMemberThread.mockResolvedValue(Ok(thread)); + store.listSubAgents.mockResolvedValue(Ok([])); + const result = (await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap(); + expect(store.ensureMemberThread).toHaveBeenCalledWith({orgId: 'org-1', agentId: 'login-1', personaId: 'persona-1', userId: 'user-1'}); + expect(result.threads.map((candidate) => candidate.id)).toEqual(['thread-1']); }); }); diff --git a/packages/api/src/harness/inspect-route.ts b/packages/api/src/harness/inspect-route.ts index bc3f4844..204a3dba 100644 --- a/packages/api/src/harness/inspect-route.ts +++ b/packages/api/src/harness/inspect-route.ts @@ -10,7 +10,7 @@ import type {AuthenticatedRequest} from '../server/middleware/require-auth'; import {type OrgViewer, requireViewer} from '../utils/user-org'; import {COMPACT_AT_TOKENS, compactionCut, type SubAgent, toMessages, toTranscript} from './context'; import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from './inference'; -import {toPersona} from './route'; +import {requireThread, toPersona} from './route'; import type {HarnessStore} from './store'; import {toolsFor} from './tool-surface'; @@ -26,7 +26,7 @@ import {toolsFor} from './tool-surface'; /** What the inspector's routes need of the store. */ export type InspectStore = Pick< HarnessStore, - 'getPersona' | 'getMemberThread' | 'listSubAgents' | 'listEntriesWithPending' | 'listThreadJobs' | 'listJobEvents' | 'getEntry' | 'getJobThread' | 'listEntries' + 'getPersona' | 'getMemberThread' | 'ensureMemberThread' | 'listSubAgents' | 'listEntriesWithPending' | 'listThreadJobs' | 'listJobEvents' | 'getEntry' | 'getJobThread' | 'listEntries' >; function toInspectedEntry(entry: EntryRow): InspectedEntry { @@ -118,13 +118,13 @@ function inspectThread(store: InspectStore, thread: ThreadRow, jobs: JobDetailRo }); } -/** `GET /api/orgs/:org/chat/personas/:personaId/inspect`: the viewer's conversation with the persona as the harness has it; not found before they have one. */ +/** `GET /api/orgs/:org/chat/personas/:personaId/inspect`: the viewer's conversation with the persona as the harness has it, begun where there was none, as the Chat page begins it. */ function handleInspectThreadRoute(store: InspectStore, request: AuthenticatedRequest): Promise, ApiError>> { return Do(async ($) => { const viewer = $(await requireInspector(request)); const personaId = request.params.personaId ?? ''; const persona = $(requirePresent($(await store.getPersona(viewer.orgId, personaId)), ApiErr.notFound('persona', personaId))); - const conversation = $(requirePresent($(await store.getMemberThread(persona.id, viewer.userId)), ApiErr.notFound('thread', persona.id))); + const conversation = $(await requireThread(store, viewer, persona)); const now = new Date(); const subAgents = $(await store.listSubAgents(conversation.id, now)); const threads: ThreadRow[] = [conversation, ...subAgents]; diff --git a/packages/api/src/harness/route.ts b/packages/api/src/harness/route.ts index 3868c886..33ff07a3 100644 --- a/packages/api/src/harness/route.ts +++ b/packages/api/src/harness/route.ts @@ -100,9 +100,8 @@ function requireManagedPersona(store: ChatStore, viewer: OrgViewer, request: Aut }); } -// The viewer's conversation with the persona, begun the first time, through the organization's -// built-in login, made with it where there is none yet. -function requireThread(store: ChatStore, viewer: OrgViewer, persona: PersonaRow): Promise> { +/** The viewer's conversation with the persona, begun the first time, through the organization's built-in login, made with it where there is none yet; the inspector begins it too. */ +export function requireThread(store: Pick, viewer: OrgViewer, persona: PersonaRow): Promise> { return Do(async ($) => { const existing = $(await store.getMemberThread(persona.id, viewer.userId)); if (existing) { From 3921fd8c845ba47a79daf116987e1aab281184c2 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:44:33 +0000 Subject: [PATCH 4/5] give the inspector a place of its own: one folder per package, its own store, and nothing of it in the harness --- packages/api/src/harness/fake-store.ts | 3 -- packages/api/src/harness/store.ts | 11 +---- .../route.test.ts} | 29 ++++++++--- .../inspect-route.ts => inspect/route.ts} | 22 ++++----- packages/api/src/inspect/store.ts | 30 ++++++++++++ packages/api/src/server/create-api-router.ts | 5 +- packages/db/package.json | 1 + packages/db/src/harness.ts | 48 +++--------------- packages/db/src/inspect.ts | 49 +++++++++++++++++++ .../inspect-client.ts => inspect/client.ts} | 0 .../derive.test.ts} | 2 +- .../src/{lib/inspect.ts => inspect/derive.ts} | 0 .../{pages => inspect}/inspect-page.test.tsx | 2 +- .../src/{pages => inspect}/inspect-page.tsx | 12 ++--- packages/web/src/inspect/loaders.ts | 29 +++++++++++ .../request-pane.tsx} | 4 +- .../thread-log.tsx} | 2 +- .../thread-tree.tsx} | 2 +- packages/web/src/loaders.ts | 31 ++---------- packages/web/src/router.tsx | 5 +- 20 files changed, 168 insertions(+), 119 deletions(-) rename packages/api/src/{harness/inspect-route.test.ts => inspect/route.test.ts} (91%) rename packages/api/src/{harness/inspect-route.ts => inspect/route.ts} (93%) create mode 100644 packages/api/src/inspect/store.ts create mode 100644 packages/db/src/inspect.ts rename packages/web/src/{client/inspect-client.ts => inspect/client.ts} (100%) rename packages/web/src/{lib/inspect.test.ts => inspect/derive.test.ts} (99%) rename packages/web/src/{lib/inspect.ts => inspect/derive.ts} (100%) rename packages/web/src/{pages => inspect}/inspect-page.test.tsx (99%) rename packages/web/src/{pages => inspect}/inspect-page.tsx (95%) create mode 100644 packages/web/src/inspect/loaders.ts rename packages/web/src/{components/inspect-request.tsx => inspect/request-pane.tsx} (98%) rename packages/web/src/{components/inspect-log.tsx => inspect/thread-log.tsx} (99%) rename packages/web/src/{components/inspect-threads.tsx => inspect/thread-tree.tsx} (99%) diff --git a/packages/api/src/harness/fake-store.ts b/packages/api/src/harness/fake-store.ts index ef598bbc..816ae89b 100644 --- a/packages/api/src/harness/fake-store.ts +++ b/packages/api/src/harness/fake-store.ts @@ -36,8 +36,5 @@ export function fakeStore(): FakeStore { listShownEntries: mock(), listOpenJobs: mock(), appendPersonMessage: mock(), - listEntriesWithPending: mock(), - listThreadJobs: mock(), - listJobEvents: mock(), }; } diff --git a/packages/api/src/harness/store.ts b/packages/api/src/harness/store.ts index b9860839..2a9edddb 100644 --- a/packages/api/src/harness/store.ts +++ b/packages/api/src/harness/store.ts @@ -3,7 +3,7 @@ import type {Result} from 'ts-results-es'; import type {ApiError, ChatModel} from '@proxy/utils'; import {readyForInference} from '@proxy/utils/harness'; import * as harness from '@proxy/db/harness'; -import type {AfterAnswer, AssistantTurn, EntryRow, FailureNote, JobDetailRow, JobEventRow, JobRow, LastMessage, Persona, PersonaRow, SubAgentRow, ThreadRow} from '@proxy/db/harness'; +import type {AfterAnswer, AssistantTurn, EntryRow, FailureNote, JobRow, LastMessage, Persona, PersonaRow, SubAgentRow, ThreadRow} from '@proxy/db/harness'; import {type AgentTaskRow, createAgentTask, getAgentTask, type NewAgentTask} from '@proxy/db/task'; import type {SignedInAgent} from '@proxy/db/agent'; import {getOrganizationChatModel} from '@proxy/db/organization'; @@ -47,11 +47,6 @@ export type HarnessStore = { listShownEntries: (threadId: string) => Promise>; listOpenJobs: (threadId: string) => Promise>; appendPersonMessage: (threadId: string, text: string) => Promise>; - - // The inspector's reads: a thread's log whole, every job of a conversation and what happened to each. - listEntriesWithPending: (threadId: string) => Promise>; - listThreadJobs: (threadIds: string[]) => Promise>; - listJobEvents: (threadIds: string[]) => Promise>; }; /** The store in the database. */ @@ -86,8 +81,4 @@ export const harnessStore: HarnessStore = { listShownEntries: harness.listShownEntries, listOpenJobs: harness.listOpenJobs, appendPersonMessage: (threadId, text) => harness.appendPersonMessage(threadId, text, readyForInference), - - listEntriesWithPending: harness.listEntriesWithPending, - listThreadJobs: harness.listThreadJobs, - listJobEvents: harness.listJobEvents, }; diff --git a/packages/api/src/harness/inspect-route.test.ts b/packages/api/src/inspect/route.test.ts similarity index 91% rename from packages/api/src/harness/inspect-route.test.ts rename to packages/api/src/inspect/route.test.ts index 714b008b..04832812 100644 --- a/packages/api/src/harness/inspect-route.test.ts +++ b/packages/api/src/inspect/route.test.ts @@ -1,4 +1,4 @@ -import {beforeEach, describe, expect, mock, test} from 'bun:test'; +import {beforeEach, describe, expect, type Mock, mock, test} from 'bun:test'; import {Err, Ok} from 'ts-results-es'; import {ApiErr} from '@proxy/utils'; @@ -11,16 +11,17 @@ mock.module('../mcp/tools', () => ({agentInstructions, MCP_TOOLS: []})); const delegate = {name: 'delegate', description: 'Delegates', parameters: {}}; const read = {name: 'read', description: 'Reads', parameters: {}}; -mock.module('./tool-surface', () => ({toolsFor: (candidate: {parentThreadId: string | null}) => (candidate.parentThreadId === null ? [delegate] : [read])})); +mock.module('../harness/tool-surface', () => ({toolsFor: (candidate: {parentThreadId: string | null}) => (candidate.parentThreadId === null ? [delegate] : [read])})); -import {type FakeStore, fakeStore} from './fake-store'; +import {fakeStore} from '../harness/fake-store'; +import type {InspectStore} from './store'; const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'debug']}; const requireViewer = mock(async () => Ok(viewer)); mock.module('../utils/user-org', () => ({requireViewer})); const ensureBuiltInAgent = mock(async () => Ok({id: 'login-1'})); -mock.module('./built-in-agent', () => ({ensureBuiltInAgent})); +mock.module('../harness/built-in-agent', () => ({ensureBuiltInAgent})); const now = new Date('2026-10-21T10:00:00Z'); const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; @@ -123,13 +124,25 @@ function request(params: Record = {}) { return {user: {userId: 'user-1', email: 'a@example.com'}, params, body: undefined} as never; } -let store: FakeStore; -let routes: ReturnType; +// The harness's fake store, with the inspector's own reads as mocks beside it. +type FakeInspectStore = {[K in keyof InspectStore]: Mock}; + +function fakeInspectStore(): FakeInspectStore { + return { + ...fakeStore(), + listJobEvents: mock(), + listThreadJobs: mock(), + listEntriesWithPending: mock(), + }; +} + +let store: FakeInspectStore; +let routes: ReturnType; beforeEach(async () => { mock.clearAllMocks(); requireViewer.mockResolvedValue(Ok(viewer)); - store = fakeStore(); + store = fakeInspectStore(); store.getPersona.mockResolvedValue(Ok(persona)); store.getMemberThread.mockResolvedValue(Ok(thread)); store.listSubAgents.mockResolvedValue(Ok([{...sub, state: {kind: 'stopped'}}])); @@ -142,7 +155,7 @@ beforeEach(async () => { return found ? Ok(found) : Err(ApiErr.notFound('entry', id)); }); store.getJobThread.mockImplementation(async (id) => Ok(id === 'thread-1' ? thread : id === 'thread-2' ? sub : null)); - const {inspectRoutes} = await import('./inspect-route'); + const {inspectRoutes} = await import('./route'); routes = inspectRoutes(store); }); diff --git a/packages/api/src/harness/inspect-route.ts b/packages/api/src/inspect/route.ts similarity index 93% rename from packages/api/src/harness/inspect-route.ts rename to packages/api/src/inspect/route.ts index 204a3dba..42e0151e 100644 --- a/packages/api/src/harness/inspect-route.ts +++ b/packages/api/src/inspect/route.ts @@ -3,32 +3,28 @@ import type {z} from 'zod'; import {ApiErr, type ApiError, Do, hasExperiment, requirePresent} from '@proxy/utils'; import {pendingToolCalls, readyForInference} from '@proxy/utils/harness'; -import type {EntryRow, JobDetailRow, JobEventRow, ThreadRow} from '@proxy/db/harness'; +import type {EntryRow, JobEventRow, ThreadRow} from '@proxy/db/harness'; +import type {JobDetailRow} from '@proxy/db/inspect'; import type {InspectedEntry, InspectedJob, InspectedJobEvent, InspectedThread, inspectionSchema, ModelRequest, ThreadState} from '@proxy/contracts/inspect'; import type {AuthenticatedRequest} from '../server/middleware/require-auth'; import {type OrgViewer, requireViewer} from '../utils/user-org'; -import {COMPACT_AT_TOKENS, compactionCut, type SubAgent, toMessages, toTranscript} from './context'; -import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from './inference'; -import {requireThread, toPersona} from './route'; -import type {HarnessStore} from './store'; -import {toolsFor} from './tool-surface'; +import {COMPACT_AT_TOKENS, compactionCut, type SubAgent, toMessages, toTranscript} from '../harness/context'; +import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from '../harness/inference'; +import {requireThread, toPersona} from '../harness/route'; +import type {InspectStore} from './store'; +import {toolsFor} from '../harness/tool-surface'; /** * The inspector's routes, for whoever works on the built-in agent, behind the `debug` experiment * beside `harness`: the viewer's conversation with a persona as the harness has it, every thread * of it with its whole log, its jobs and what happened to each, and, for any answer of the model, - * the request it was sent, rebuilt from the log with the functions the harness runs (context.ts) + * the request it was sent, rebuilt from the log with the functions the harness runs + * (../harness/context.ts) * and the system prompt the entry kept. The viewer sees their own conversations alone, as on the * Chat page: a conversation is theirs, whoever they are in the organization. */ -/** What the inspector's routes need of the store. */ -export type InspectStore = Pick< - HarnessStore, - 'getPersona' | 'getMemberThread' | 'ensureMemberThread' | 'listSubAgents' | 'listEntriesWithPending' | 'listThreadJobs' | 'listJobEvents' | 'getEntry' | 'getJobThread' | 'listEntries' ->; - function toInspectedEntry(entry: EntryRow): InspectedEntry { return { id: entry.id, diff --git a/packages/api/src/inspect/store.ts b/packages/api/src/inspect/store.ts new file mode 100644 index 00000000..2d28d91f --- /dev/null +++ b/packages/api/src/inspect/store.ts @@ -0,0 +1,30 @@ +import * as harness from '@proxy/db/harness'; +import * as inspect from '@proxy/db/inspect'; + +import type {HarnessStore} from '../harness/store'; +import {harnessStore} from '../harness/store'; + +/** + * What the inspector reads: the harness's own reads it shares (../harness/store), and its own, + * every job of a conversation and a thread's log whole (@proxy/db/inspect). The harness knows + * nothing of it: the inspector reads the harness, never the other way. + */ +export type InspectStore = Pick & { + listJobEvents: typeof harness.listJobEvents; + listThreadJobs: typeof inspect.listThreadJobs; + listEntriesWithPending: typeof inspect.listEntriesWithPending; +}; + +/** The store in the database. */ +export const inspectStore: InspectStore = { + getPersona: harnessStore.getPersona, + getMemberThread: harnessStore.getMemberThread, + ensureMemberThread: harnessStore.ensureMemberThread, + listSubAgents: harnessStore.listSubAgents, + getEntry: harnessStore.getEntry, + getJobThread: harnessStore.getJobThread, + listEntries: harnessStore.listEntries, + listJobEvents: harness.listJobEvents, + listThreadJobs: inspect.listThreadJobs, + listEntriesWithPending: inspect.listEntriesWithPending, +}; diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index c530ee92..770a5e20 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -42,9 +42,10 @@ import { handleShareConnectionRoute, } from '../connections/route'; import {handleFaviconRoute} from '../favicons/route'; -import {inspectRoutes} from '../harness/inspect-route'; import {chatRoutes} from '../harness/route'; import {harnessStore} from '../harness/store'; +import {inspectRoutes} from '../inspect/route'; +import {inspectStore} from '../inspect/store'; import {handleFillIntentRoute, handlePickIntentRoute} from '../intent/route'; import {handleMcpMethodNotAllowedRoute, handleMcpRoute} from '../mcp/route'; import {handleDeleteMeRoute, handleMeRoute} from '../me/route'; @@ -209,7 +210,7 @@ export function createApiRouter(): express.Router { org.post('/chat/personas/:personaId/messages', withAuthResult('Post chat message route', chat.handlePostMessageRoute)); // The inspector, behind the `debug` experiment too: the conversation as the harness has it, // and what the model was sent for an answer. - const inspect = inspectRoutes(harnessStore); + const inspect = inspectRoutes(inspectStore); org.get('/chat/personas/:personaId/inspect', withAuthResult('Inspect chat thread route', inspect.handleInspectThreadRoute)); org.get('/chat/inspect/entries/:entryId/request', withAuthResult('Inspect chat request route', inspect.handleInspectRequestRoute)); diff --git a/packages/db/package.json b/packages/db/package.json index 6a75477c..86a19484 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,6 +12,7 @@ "./desktop-auth": "./src/desktop-auth.ts", "./feedback": "./src/feedback.ts", "./harness": "./src/harness.ts", + "./inspect": "./src/inspect.ts", "./vault": "./src/vault.ts", "./invitation": "./src/invitation.ts", "./instance-meta": "./src/instance-meta.ts", diff --git a/packages/db/src/harness.ts b/packages/db/src/harness.ts index bb3ed395..500f54ea 100644 --- a/packages/db/src/harness.ts +++ b/packages/db/src/harness.ts @@ -211,7 +211,8 @@ function toPersona(persona: { return {...persona, look: parseLook(persona.look)}; } -const entrySelect = { +/** The columns of an entry, as `EntryRow` has them; the inspector reads with it too. */ +export const entrySelect = { id: true, seq: true, threadId: true, @@ -229,7 +230,8 @@ const entrySelect = { createdAt: true, } as const; -const jobSelect = { +/** The columns of a job the worker reads; the inspector reads more. */ +export const jobSelect = { id: true, threadId: true, kind: true, @@ -251,11 +253,12 @@ const jobEventSelect = { at: true, } as const; -type EntryRecord = Prisma.AgentEntryGetPayload<{select: typeof entrySelect}>; +export type EntryRecord = Prisma.AgentEntryGetPayload<{select: typeof entrySelect}>; type Tx = Prisma.TransactionClient; -function toEntry(entry: EntryRecord): EntryRow { +/** An entry as read, its calls parsed. */ +export function toEntry(entry: EntryRecord): EntryRow { return { ...entry, toolCalls: entry.toolCalls === null ? null : (JSON.parse(entry.toolCalls) as ToolCall[]), @@ -549,43 +552,6 @@ export async function listJobEvents(threadIds: string[]): Promise> { - return wrapDb(() => - db.agentJob.findMany({ - where: {threadId: {in: threadIds}}, - select: jobDetailSelect, - orderBy: {createdAt: 'asc'}, - }), - ); -} - -/** The thread's whole log, for the inspector: what the model reads in order, then what is pending, in the order it landed. */ -export async function listEntriesWithPending( - threadId: string, -): Promise> { - return ( - await wrapDb(() => - db.agentEntry.findMany({ - where: {threadId}, - select: entrySelect, - orderBy: [{pending: 'asc'}, {seq: 'asc'}], - }), - ) - ).map((entries) => entries.map(toEntry)); -} - /** A new task for a sub-agent's calls, once the one before was finished while it was at work. */ export async function setThreadTask( threadId: string, diff --git a/packages/db/src/inspect.ts b/packages/db/src/inspect.ts new file mode 100644 index 00000000..056c6bff --- /dev/null +++ b/packages/db/src/inspect.ts @@ -0,0 +1,49 @@ +import type {Result} from 'ts-results-es'; + +import {type ApiError, wrapDb} from '@proxy/utils'; + +import {db} from '.'; +import {type EntryRow, entrySelect, type JobRow, jobSelect, toEntry} from './harness'; + +/** + * What the inspector reads of the built-in agent beyond what the harness does (./harness): every + * job of a conversation whatever its status, and a thread's log with what is pending. Reads + * only; the harness alone writes. + */ + +/** A job with its whole record: beside what the worker reads, why it failed, its lease and when it was made. */ +export type JobDetailRow = JobRow & { + leaseExpiresAt: Date | null; + error: string | null; + createdAt: Date; +}; + +const jobDetailSelect = {...jobSelect, leaseExpiresAt: true, error: true, createdAt: true} as const; + +/** Every job of the threads, whatever its status, oldest first: the queue and its history. */ +export async function listThreadJobs( + threadIds: string[], +): Promise> { + return wrapDb(() => + db.agentJob.findMany({ + where: {threadId: {in: threadIds}}, + select: jobDetailSelect, + orderBy: {createdAt: 'asc'}, + }), + ); +} + +/** The thread's whole log: what the model reads in order, then what is pending, in the order it landed. */ +export async function listEntriesWithPending( + threadId: string, +): Promise> { + return ( + await wrapDb(() => + db.agentEntry.findMany({ + where: {threadId}, + select: entrySelect, + orderBy: [{pending: 'asc'}, {seq: 'asc'}], + }), + ) + ).map((entries) => entries.map(toEntry)); +} diff --git a/packages/web/src/client/inspect-client.ts b/packages/web/src/inspect/client.ts similarity index 100% rename from packages/web/src/client/inspect-client.ts rename to packages/web/src/inspect/client.ts diff --git a/packages/web/src/lib/inspect.test.ts b/packages/web/src/inspect/derive.test.ts similarity index 99% rename from packages/web/src/lib/inspect.test.ts rename to packages/web/src/inspect/derive.test.ts index 10550862..42a83483 100644 --- a/packages/web/src/lib/inspect.test.ts +++ b/packages/web/src/inspect/derive.test.ts @@ -8,7 +8,7 @@ import { lastAttempt, threadStatus, tokensOf, -} from './inspect'; +} from './derive'; const now = '2026-10-21T10:00:00.000Z'; diff --git a/packages/web/src/lib/inspect.ts b/packages/web/src/inspect/derive.ts similarity index 100% rename from packages/web/src/lib/inspect.ts rename to packages/web/src/inspect/derive.ts diff --git a/packages/web/src/pages/inspect-page.test.tsx b/packages/web/src/inspect/inspect-page.test.tsx similarity index 99% rename from packages/web/src/pages/inspect-page.test.tsx rename to packages/web/src/inspect/inspect-page.test.tsx index dd258d6d..5449bcb6 100644 --- a/packages/web/src/pages/inspect-page.test.tsx +++ b/packages/web/src/inspect/inspect-page.test.tsx @@ -14,7 +14,7 @@ import type { Inspection, ModelRequest, } from '@proxy/contracts/inspect'; -import {InspectPage} from '@/pages/inspect-page'; +import {InspectPage} from '@/inspect/inspect-page'; import {testHuman} from '@/test-org'; const now = '2026-10-21T10:00:00.000Z'; diff --git a/packages/web/src/pages/inspect-page.tsx b/packages/web/src/inspect/inspect-page.tsx similarity index 95% rename from packages/web/src/pages/inspect-page.tsx rename to packages/web/src/inspect/inspect-page.tsx index 57c9e65e..399a703f 100644 --- a/packages/web/src/pages/inspect-page.tsx +++ b/packages/web/src/inspect/inspect-page.tsx @@ -10,16 +10,16 @@ import { } from '@proxy/ui/components/input-group'; import {postMessage} from '@/client/harness-client'; import type {Inspection} from '@proxy/contracts/inspect'; -import {getInspection} from '@/client/inspect-client'; +import {getInspection} from '@/inspect/client'; import {Head} from '@/components/head'; -import {ThreadLog} from '@/components/inspect-log'; -import {RequestPane} from '@/components/inspect-request'; -import {JobQueue, StatusBadge, ThreadTree} from '@/components/inspect-threads'; +import {ThreadLog} from '@/inspect/thread-log'; +import {RequestPane} from '@/inspect/request-pane'; +import {JobQueue, StatusBadge, ThreadTree} from '@/inspect/thread-tree'; import {useOrgSlug} from '@/hooks/use-org'; import {formatTime} from '@/lib/format'; -import {threadLabel, threadStatus} from '@/lib/inspect'; +import {threadLabel, threadStatus} from '@/inspect/derive'; import {describeFetchError} from '@/lib/loader-utils'; -import type {inspectLoader} from '@/loaders'; +import type {inspectLoader} from '@/inspect/loaders'; /** * The inspector: the person's conversation with a persona as the harness has it, on a page of diff --git a/packages/web/src/inspect/loaders.ts b/packages/web/src/inspect/loaders.ts new file mode 100644 index 00000000..781ad68e --- /dev/null +++ b/packages/web/src/inspect/loaders.ts @@ -0,0 +1,29 @@ +import {redirect} from 'react-router'; +import {listPersonas} from '@/client/harness-client'; +import {getInspection} from '@/inspect/client'; +import {unwrapLoaderResult} from '@/lib/loader-utils'; +import {type OrgParams, orgOf} from '@/loaders'; + +// The inspector of the conversation with a persona: the first persona's where the address names none. +export async function inspectIndexLoader(args: OrgParams) { + const org = orgOf(args); + const {personas} = unwrapLoaderResult(await listPersonas(org)); + const first = personas[0]; + if (!first) { + throw redirect(`/${org}`); + } + throw redirect(`/${org}/inspect/${first.id}`); +} + +// The conversation with a persona as the harness has it, and the personas to switch between. +export async function inspectLoader(args: OrgParams & {params: {id?: string}}) { + const org = orgOf(args); + const [inspection, personas] = await Promise.all([ + getInspection(org, args.params.id ?? ''), + listPersonas(org), + ]); + return { + inspection: unwrapLoaderResult(inspection), + personas: unwrapLoaderResult(personas).personas, + }; +} diff --git a/packages/web/src/components/inspect-request.tsx b/packages/web/src/inspect/request-pane.tsx similarity index 98% rename from packages/web/src/components/inspect-request.tsx rename to packages/web/src/inspect/request-pane.tsx index f151b9e2..6132e811 100644 --- a/packages/web/src/components/inspect-request.tsx +++ b/packages/web/src/inspect/request-pane.tsx @@ -7,10 +7,10 @@ import type { ModelMessage, ModelRequest, } from '@proxy/contracts/inspect'; -import {getRequest} from '@/client/inspect-client'; +import {getRequest} from '@/inspect/client'; import {useOrgSlug} from '@/hooks/use-org'; import {formatTime} from '@/lib/format'; -import {formatDuration, prettyJson, timedEvents} from '@/lib/inspect'; +import {formatDuration, prettyJson, timedEvents} from '@/inspect/derive'; // The inspector's right column: what the model was sent for the answer picked in the log, as the // api rebuilds it, the system prompt, the messages and the tools; what came back; and what diff --git a/packages/web/src/components/inspect-log.tsx b/packages/web/src/inspect/thread-log.tsx similarity index 99% rename from packages/web/src/components/inspect-log.tsx rename to packages/web/src/inspect/thread-log.tsx index c9d6af07..7658ef4c 100644 --- a/packages/web/src/components/inspect-log.tsx +++ b/packages/web/src/inspect/thread-log.tsx @@ -3,7 +3,7 @@ import {Button} from '@proxy/ui/components/button'; import {cn} from '@proxy/ui/lib/utils'; import type {InspectedEntry, InspectedThread, ThreadState} from '@proxy/contracts/inspect'; import {formatTime} from '@/lib/format'; -import {coveredSeq, prettyJson} from '@/lib/inspect'; +import {coveredSeq, prettyJson} from '@/inspect/derive'; // The inspector's middle: a thread's whole log as the harness has it, every entry with what the // model doesn't see (its seq, its cost, its job), and above it where the thread stands for the diff --git a/packages/web/src/components/inspect-threads.tsx b/packages/web/src/inspect/thread-tree.tsx similarity index 99% rename from packages/web/src/components/inspect-threads.tsx rename to packages/web/src/inspect/thread-tree.tsx index 445aeb43..370687a7 100644 --- a/packages/web/src/components/inspect-threads.tsx +++ b/packages/web/src/inspect/thread-tree.tsx @@ -13,7 +13,7 @@ import { threadStatus, type ThreadStatus, tokensOf, -} from '@/lib/inspect'; +} from '@/inspect/derive'; // The inspector's left column: the conversation's threads, the conversation first and the // sub-agents started from it under it, each with where it stands; and under them the queue, diff --git a/packages/web/src/loaders.ts b/packages/web/src/loaders.ts index 6011f238..ed0b33f3 100644 --- a/packages/web/src/loaders.ts +++ b/packages/web/src/loaders.ts @@ -10,7 +10,6 @@ import { } from '@/client/agents-client'; import {getConnection, listConnections} from '@/client/connections-client'; import {getThread, listPersonas} from '@/client/harness-client'; -import {getInspection} from '@/client/inspect-client'; import {getMe} from '@/client/me-client'; import {getAuthorizationRequest} from '@/client/oauth-client'; import {getInvitation, getOrg, listMembers, listOrgs} from '@/client/orgs-client'; @@ -27,9 +26,11 @@ import {redirect, type ShouldRevalidateFunctionArgs} from 'react-router'; // page is in an organization, named by its slug in the address (`params.org`), which its api // calls are about. -type OrgParams = {params: {org?: string}}; +/** The route params of a page in an organization. */ +export type OrgParams = {params: {org?: string}}; -function orgOf({params}: OrgParams): string { +/** The organization's slug from the address. */ +export function orgOf({params}: OrgParams): string { return params.org ?? ''; } @@ -135,30 +136,6 @@ export async function chatLoader(args: OrgParams & {params: {id?: string}}) { return {log: unwrapLoaderResult(log), personas: unwrapLoaderResult(personas).personas}; } -// The inspector of the conversation with a persona: the first persona's where the address names none. -export async function inspectIndexLoader(args: OrgParams) { - const org = orgOf(args); - const {personas} = unwrapLoaderResult(await listPersonas(org)); - const first = personas[0]; - if (!first) { - throw redirect(`/${org}`); - } - throw redirect(`/${org}/inspect/${first.id}`); -} - -// The conversation with a persona as the harness has it, and the personas to switch between. -export async function inspectLoader(args: OrgParams & {params: {id?: string}}) { - const org = orgOf(args); - const [inspection, personas] = await Promise.all([ - getInspection(org, args.params.id ?? ''), - listPersonas(org), - ]); - return { - inspection: unwrapLoaderResult(inspection), - personas: unwrapLoaderResult(personas).personas, - }; -} - // The vault is the one built-in connection, which every organization has twice: the person's own // there and the organization's, shared with everyone in it. The entries of each are kept in // Personal Agent Proxy. diff --git a/packages/web/src/router.tsx b/packages/web/src/router.tsx index 98999c96..d4ce28c5 100644 --- a/packages/web/src/router.tsx +++ b/packages/web/src/router.tsx @@ -31,8 +31,6 @@ import { connectionsLoader, desktopAuthLoader, homeLoader, - inspectIndexLoader, - inspectLoader, inviteLoader, membersLoader, shouldRevalidateHome, @@ -67,7 +65,8 @@ import {ConnectionsPage} from '@/pages/connections-page'; import {DesktopAuthPage} from '@/pages/desktop-auth-page'; import {DesktopReturnPage} from '@/pages/desktop-return-page'; import {HomePage} from '@/pages/home-page'; -import {InspectPage} from '@/pages/inspect-page'; +import {inspectIndexLoader, inspectLoader} from '@/inspect/loaders'; +import {InspectPage} from '@/inspect/inspect-page'; import {InvitePage} from '@/pages/invite-page'; import {LoginPage} from '@/pages/login-page'; import {MembersPage} from '@/pages/members-page'; From 97def0852e6088a40a321f991c0a2bcf5ef24251 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:07:15 +0000 Subject: [PATCH 5/5] move how the model reads a log into utils, so the admin can rebuild what it was sent, and take the inspector out: it goes to internal --- packages/api/src/harness/inference.test.ts | 2 +- packages/api/src/harness/inference.ts | 13 +- packages/api/src/harness/route.ts | 8 +- packages/api/src/inspect/route.test.ts | 245 ------------- packages/api/src/inspect/route.ts | 191 ---------- packages/api/src/inspect/store.ts | 30 -- packages/api/src/llm/open-router.ts | 11 +- packages/api/src/server/create-api-router.ts | 7 - packages/contracts/src/inspect.ts | 158 --------- packages/db/package.json | 1 - packages/db/src/harness.test.ts | 6 +- packages/db/src/harness.ts | 11 +- packages/db/src/inspect.ts | 49 --- packages/utils/package.json | 1 + packages/utils/src/experiments.ts | 3 - .../src/harness-context.test.ts} | 151 ++++++-- .../src/harness-context.ts} | 103 ++++-- packages/web/src/inspect/client.ts | 17 - packages/web/src/inspect/derive.test.ts | 186 ---------- packages/web/src/inspect/derive.ts | 144 -------- .../web/src/inspect/inspect-page.test.tsx | 327 ------------------ packages/web/src/inspect/inspect-page.tsx | 218 ------------ packages/web/src/inspect/loaders.ts | 29 -- packages/web/src/inspect/request-pane.tsx | 207 ----------- packages/web/src/inspect/thread-log.tsx | 196 ----------- packages/web/src/inspect/thread-tree.tsx | 250 ------------- packages/web/src/loaders.ts | 6 +- packages/web/src/router.tsx | 6 - 28 files changed, 224 insertions(+), 2352 deletions(-) delete mode 100644 packages/api/src/inspect/route.test.ts delete mode 100644 packages/api/src/inspect/route.ts delete mode 100644 packages/api/src/inspect/store.ts delete mode 100644 packages/contracts/src/inspect.ts delete mode 100644 packages/db/src/inspect.ts rename packages/{api/src/harness/context.test.ts => utils/src/harness-context.test.ts} (65%) rename packages/{api/src/harness/context.ts => utils/src/harness-context.ts} (62%) delete mode 100644 packages/web/src/inspect/client.ts delete mode 100644 packages/web/src/inspect/derive.test.ts delete mode 100644 packages/web/src/inspect/derive.ts delete mode 100644 packages/web/src/inspect/inspect-page.test.tsx delete mode 100644 packages/web/src/inspect/inspect-page.tsx delete mode 100644 packages/web/src/inspect/loaders.ts delete mode 100644 packages/web/src/inspect/request-pane.tsx delete mode 100644 packages/web/src/inspect/thread-log.tsx delete mode 100644 packages/web/src/inspect/thread-tree.tsx diff --git a/packages/api/src/harness/inference.test.ts b/packages/api/src/harness/inference.test.ts index cef9e6c1..34413f02 100644 --- a/packages/api/src/harness/inference.test.ts +++ b/packages/api/src/harness/inference.test.ts @@ -329,7 +329,7 @@ describe('runInference', () => { test('a log grown long is summarized first, and the model reads the summary in place of what it covers', async () => { const {runInference} = await import('./inference'); - const {COMPACT_AT_TOKENS} = await import('./context'); + const {COMPACT_AT_TOKENS} = await import('@proxy/utils/harness-context'); const long = [ entry('person', 'One'), entry('assistant', '', {toolCalls: [{id: 'c1', name: 'read', arguments: '{}'}], inputTokens: 10}), diff --git a/packages/api/src/harness/inference.ts b/packages/api/src/harness/inference.ts index b13309d6..4fac357e 100644 --- a/packages/api/src/harness/inference.ts +++ b/packages/api/src/harness/inference.ts @@ -7,7 +7,7 @@ import {askForTurn, type ModelTurn} from '../llm/open-router'; import {agentInstructions} from '../mcp/tools'; import {log, serializeError} from '../observability/log'; import {env} from '../utils/env'; -import {compactionCut, toMessages, toTranscript} from './context'; +import {compactionCut, toMessages, toTranscript} from '@proxy/utils/harness-context'; import {afterAnswer, planCalls} from './conversation'; import type {HarnessStore} from './store'; import {toolsFor} from './tool-surface'; @@ -50,15 +50,15 @@ const SUB_FLOW = // How a long log is summarized, by the same model, for itself: a conversation's, or a // sub-agent's, whose messages from the person are from the agent it works for. -export const SUMMARIZE = +const SUMMARIZE = 'You are summarizing a conversation between a person and their agent, written out below with the calls the agent made and what they answered, so that the agent can go on from the summary in place of the conversation so far. Write what the agent needs to go on as if it remembered: what the person asked for and what they said they want in general; what was done and found, with the names, ids, dates and numbers that matter; what the person was told; what is still open, promised or planned, such as what to do after a sleep or when a sub-agent answers, and which sub-agents, by id, are on what. Leave out what no longer matters. Plain prose, from the agent’s side, no title.'; -export const SUMMARIZE_NOW = 'Write the summary now.'; +const SUMMARIZE_NOW = 'Write the summary now.'; // How many answers the model may give in a row without the person saying anything, calls, // sub-agents' messages and wake-ups included, before it stops and waits for them; a message that // comes after the stop is acted on, and the count starts over. For a sub-agent, the person is the // agent it works for, whose messages are what it hears. -export const MAX_TURNS_ALONE = 30; +const MAX_TURNS_ALONE = 30; // How often a model that can't be reached is tried, and how long to wait between tries. const MAX_MODEL_ATTEMPTS = 3; @@ -81,8 +81,9 @@ function isStop(entry: EntryRow): boolean { return entry.kind === 'assistant' && (entry.text === TOO_LONG || entry.text === TOO_LONG_SUB); } -/** How many answers the model has given since the person last spoke, or since it last stopped for them. */ -export function turnsAlone(entries: EntryRow[]): number { +// How many answers the model has given since the person last spoke, or since it last stopped for +// them. +function turnsAlone(entries: EntryRow[]): number { const since = entries.findLastIndex((entry) => entry.kind === 'person' || isStop(entry)); return entries.slice(since + 1).filter((entry) => entry.kind === 'assistant').length; } diff --git a/packages/api/src/harness/route.ts b/packages/api/src/harness/route.ts index 33ff07a3..52d25697 100644 --- a/packages/api/src/harness/route.ts +++ b/packages/api/src/harness/route.ts @@ -49,8 +49,7 @@ function managesPersona(viewer: OrgViewer, persona: PersonaRow): boolean { return manages(viewer, {ownerUserId: null, byId: persona.createdById}); } -/** A persona for a response, with whether the viewer may change it; the inspector's too. */ -export function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): ChatPersona { +function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): ChatPersona { const last = persona.lastMessage ?? null; return { id: persona.id, @@ -100,8 +99,9 @@ function requireManagedPersona(store: ChatStore, viewer: OrgViewer, request: Aut }); } -/** The viewer's conversation with the persona, begun the first time, through the organization's built-in login, made with it where there is none yet; the inspector begins it too. */ -export function requireThread(store: Pick, viewer: OrgViewer, persona: PersonaRow): Promise> { +// The viewer's conversation with the persona, begun the first time, through the organization's +// built-in login, made with it where there is none yet. +function requireThread(store: ChatStore, viewer: OrgViewer, persona: PersonaRow): Promise> { return Do(async ($) => { const existing = $(await store.getMemberThread(persona.id, viewer.userId)); if (existing) { diff --git a/packages/api/src/inspect/route.test.ts b/packages/api/src/inspect/route.test.ts deleted file mode 100644 index 04832812..00000000 --- a/packages/api/src/inspect/route.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -import {beforeEach, describe, expect, type Mock, mock, test} from 'bun:test'; -import {Err, Ok} from 'ts-results-es'; - -import {ApiErr} from '@proxy/utils'; -import type {EntryRow, ThreadRow} from '@proxy/db/harness'; - -mock.module('../utils/env', () => ({env: {NODE_ENV: 'test', APP_URL: 'http://localhost:5173'}})); - -const agentInstructions = mock(async (_agent: unknown, flow: string) => Ok(`You are signed in.\n\n${flow}`)); -mock.module('../mcp/tools', () => ({agentInstructions, MCP_TOOLS: []})); - -const delegate = {name: 'delegate', description: 'Delegates', parameters: {}}; -const read = {name: 'read', description: 'Reads', parameters: {}}; -mock.module('../harness/tool-surface', () => ({toolsFor: (candidate: {parentThreadId: string | null}) => (candidate.parentThreadId === null ? [delegate] : [read])})); - -import {fakeStore} from '../harness/fake-store'; -import type {InspectStore} from './store'; - -const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'debug']}; -const requireViewer = mock(async () => Ok(viewer)); -mock.module('../utils/user-org', () => ({requireViewer})); - -const ensureBuiltInAgent = mock(async () => Ok({id: 'login-1'})); -mock.module('../harness/built-in-agent', () => ({ensureBuiltInAgent})); - -const now = new Date('2026-10-21T10:00:00Z'); -const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; -const persona = {id: 'persona-1', name: 'Finance', look, color: 'blue', createdById: 'user-1'}; -const thread: ThreadRow = { - id: 'thread-1', - orgId: 'org-1', - agentId: 'login-1', - personaId: 'persona-1', - userId: 'user-1', - parentThreadId: null, - parentCallId: null, - title: null, - taskId: null, - createdAt: now, -}; -const sub = {...thread, id: 'thread-2', parentThreadId: 'thread-1', parentCallId: 'c1', title: 'Find the invoices', taskId: 'task-2'}; - -type Kind = 'person' | 'assistant' | 'tool_result' | 'event' | 'summary'; - -function entry( - seq: number, - threadId: string, - kind: Kind, - text: string, - extra: Partial<{toolCalls: Array<{id: string; name: string; arguments: string}>; callId: string; inputTokens: number; coversSeq: number; system: string; jobId: string; pending: boolean}> = {}, -) { - return { - id: `e${seq}`, - seq, - threadId, - kind, - pending: extra.pending ?? false, - text, - toolCalls: extra.toolCalls ?? null, - callId: extra.callId ?? null, - isError: false, - inputTokens: extra.inputTokens ?? null, - outputTokens: null, - coversSeq: extra.coversSeq ?? null, - system: extra.system ?? null, - jobId: extra.jobId ?? null, - createdAt: now, - } satisfies EntryRow; -} - -// The person asked, the agent delegated, and the sub-agent has read once and is waiting on it. -const conversation = [ - entry(1, 'thread-1', 'person', 'Find my invoices'), - entry(2, 'thread-1', 'assistant', '', { - toolCalls: [{id: 'c1', name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}], - inputTokens: 1200, - system: 'You are signed in.', - jobId: 'job-1', - }), - entry(3, 'thread-1', 'tool_result', 'Started the sub-agent “Find the invoices”.', {callId: 'c1', jobId: 'job-2'}), - entry(4, 'thread-1', 'assistant', 'Looking.', {inputTokens: 1300, system: 'You are signed in.', jobId: 'job-3'}), - entry(7, 'thread-1', 'person', 'Thanks', {pending: true}), -]; -const subLog = [ - entry(5, 'thread-2', 'person', 'Find them'), - entry(6, 'thread-2', 'assistant', '', {toolCalls: [{id: 'c2', name: 'read', arguments: '{}'}], inputTokens: 800, system: 'You are a sub-agent.', jobId: 'job-4'}), -]; -const jobs = [ - { - id: 'job-1', - threadId: 'thread-1', - kind: 'inference' as const, - status: 'done' as const, - entryId: null, - callId: null, - runAt: now, - claimedBy: null, - leaseExpiresAt: null, - attempts: 1, - error: null, - createdAt: now, - }, - { - id: 'job-5', - threadId: 'thread-2', - kind: 'tool' as const, - status: 'running' as const, - entryId: 'e6', - callId: 'c2', - runAt: now, - claimedBy: 'host-1', - leaseExpiresAt: now, - attempts: 1, - error: null, - createdAt: now, - }, -]; -const events = [ - {id: 'ev1', jobId: 'job-1', threadId: 'thread-1', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, - {id: 'ev2', jobId: 'job-5', threadId: 'thread-2', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, -]; - -function request(params: Record = {}) { - return {user: {userId: 'user-1', email: 'a@example.com'}, params, body: undefined} as never; -} - -// The harness's fake store, with the inspector's own reads as mocks beside it. -type FakeInspectStore = {[K in keyof InspectStore]: Mock}; - -function fakeInspectStore(): FakeInspectStore { - return { - ...fakeStore(), - listJobEvents: mock(), - listThreadJobs: mock(), - listEntriesWithPending: mock(), - }; -} - -let store: FakeInspectStore; -let routes: ReturnType; - -beforeEach(async () => { - mock.clearAllMocks(); - requireViewer.mockResolvedValue(Ok(viewer)); - store = fakeInspectStore(); - store.getPersona.mockResolvedValue(Ok(persona)); - store.getMemberThread.mockResolvedValue(Ok(thread)); - store.listSubAgents.mockResolvedValue(Ok([{...sub, state: {kind: 'stopped'}}])); - store.listThreadJobs.mockResolvedValue(Ok(jobs)); - store.listJobEvents.mockResolvedValue(Ok(events)); - store.listEntriesWithPending.mockImplementation(async (threadId) => Ok(threadId === 'thread-1' ? conversation : subLog)); - store.listEntries.mockImplementation(async (threadId) => Ok((threadId === 'thread-1' ? conversation : subLog).filter((candidate) => !candidate.pending))); - store.getEntry.mockImplementation(async (id) => { - const found = [...conversation, ...subLog].find((candidate) => candidate.id === id); - return found ? Ok(found) : Err(ApiErr.notFound('entry', id)); - }); - store.getJobThread.mockImplementation(async (id) => Ok(id === 'thread-1' ? thread : id === 'thread-2' ? sub : null)); - const {inspectRoutes} = await import('./route'); - routes = inspectRoutes(store); -}); - -describe('the inspection of a conversation', () => { - test('is refused without the inspector experiment, as the Chat page is without its own', async () => { - requireViewer.mockResolvedValue(Ok({...viewer, experiments: ['harness']})); - expect((await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('forbidden'); - expect((await routes.handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('forbidden'); - }); - - test('is the conversation first, then its sub-agents, each with its whole log, its jobs and their events', async () => { - const result = (await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap(); - expect(result.persona.id).toBe('persona-1'); - expect(result.threads.map((candidate) => candidate.id)).toEqual(['thread-1', 'thread-2']); - expect(store.listThreadJobs).toHaveBeenCalledWith(['thread-1', 'thread-2']); - expect(store.listJobEvents).toHaveBeenCalledWith(['thread-1', 'thread-2']); - const [main, agent] = result.threads; - expect(main?.entries.map((candidate) => candidate.kind)).toEqual(['person', 'assistant', 'tool_result', 'assistant', 'person']); - expect(main?.entries.at(-1)?.pending).toBe(true); - expect(main?.entries[1]).toMatchObject({hasSystem: true, jobId: 'job-1', inputTokens: 1200, toolCalls: [{id: 'c1', name: 'delegate', arguments: expect.any(String)}]}); - expect(main?.jobs.map((candidate) => candidate.id)).toEqual(['job-1']); - expect(main?.events.map((candidate) => candidate.id)).toEqual(['ev1']); - expect(agent?.title).toBe('Find the invoices'); - expect(agent?.jobs.map((candidate) => candidate.id)).toEqual(['job-5']); - expect(agent?.jobs[0]).toMatchObject({status: 'running', claimedBy: 'host-1', leaseExpiresAt: now.toISOString()}); - }); - - test('says where each thread stands: the conversation waits for the pending turn to land, the sub-agent for its call', async () => { - const [main, agent] = (await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap().threads; - // The pending turn is not in the log yet: the log ends with an answer, so the model is not due. - expect(main?.state).toMatchObject({ready: false, pendingCalls: [], turnsAlone: 2, maxTurnsAlone: 30, lastInputTokens: 1300, compactAtTokens: 80_000, compactionCoversSeq: null}); - expect(agent?.state).toMatchObject({ready: false, pendingCalls: [{id: 'c2', name: 'read'}], turnsAlone: 1, lastInputTokens: 800}); - }); - - test('begins the conversation where the viewer has none yet, as the Chat page does', async () => { - store.getMemberThread.mockResolvedValue(Ok(null)); - store.ensureMemberThread.mockResolvedValue(Ok(thread)); - store.listSubAgents.mockResolvedValue(Ok([])); - const result = (await routes.handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap(); - expect(store.ensureMemberThread).toHaveBeenCalledWith({orgId: 'org-1', agentId: 'login-1', personaId: 'persona-1', userId: 'user-1'}); - expect(result.threads.map((candidate) => candidate.id)).toEqual(['thread-1']); - }); -}); - -describe('the request behind an entry', () => { - test('is the system prompt the answer kept, the log before it as the model read it, and the thread’s tools', async () => { - const result = (await routes.handleInspectRequestRoute(request({entryId: 'e4'}))).unwrap(); - expect(result.kind).toBe('turn'); - expect(result.system).toBe('You are signed in.'); - expect(result.messages).toEqual([ - {role: 'system', content: 'You are signed in.'}, - {role: 'user', content: 'Find my invoices'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c1', type: 'function', function: {name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}}]}, - {role: 'tool', tool_call_id: 'c1', content: 'Started the sub-agent “Find the invoices”.'}, - ]); - expect(result.tools).toEqual([delegate]); - }); - - test('has a sub-agent’s tools on a sub-agent’s thread', async () => { - const result = (await routes.handleInspectRequestRoute(request({entryId: 'e6'}))).unwrap(); - expect(result.messages).toEqual([ - {role: 'system', content: 'You are a sub-agent.'}, - {role: 'user', content: 'Find them'}, - ]); - expect(result.tools).toEqual([read]); - }); - - test('is the transcript of what a summary covers, for the model to summarize', async () => { - store.getEntry.mockResolvedValue(Ok(entry(5, 'thread-1', 'summary', 'The person asked for invoices.', {coversSeq: 3}))); - const result = (await routes.handleInspectRequestRoute(request({entryId: 'e5'}))).unwrap(); - expect(result.kind).toBe('summary'); - expect(result.tools).toEqual([]); - expect(result.messages[0]?.role).toBe('system'); - const asked = result.messages[1]; - expect(asked?.role).toBe('user'); - expect(asked?.content).toContain('Person:\nFind my invoices'); - expect(asked?.content).toContain('Agent called delegate with'); - expect(asked?.content).not.toContain('Looking.'); - expect(asked?.content).toMatch(/Write the summary now\.$/); - }); - - test('is not found for an entry of someone else’s thread, or one that is neither an answer nor a summary', async () => { - expect((await routes.handleInspectRequestRoute(request({entryId: 'e1'}))).unwrapErr().kind).toBe('not_found'); - store.getJobThread.mockResolvedValue(Ok({...thread, userId: 'user-2'})); - expect((await routes.handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('not_found'); - }); -}); diff --git a/packages/api/src/inspect/route.ts b/packages/api/src/inspect/route.ts deleted file mode 100644 index 42e0151e..00000000 --- a/packages/api/src/inspect/route.ts +++ /dev/null @@ -1,191 +0,0 @@ -import {Err, type Result} from 'ts-results-es'; -import type {z} from 'zod'; - -import {ApiErr, type ApiError, Do, hasExperiment, requirePresent} from '@proxy/utils'; -import {pendingToolCalls, readyForInference} from '@proxy/utils/harness'; -import type {EntryRow, JobEventRow, ThreadRow} from '@proxy/db/harness'; -import type {JobDetailRow} from '@proxy/db/inspect'; -import type {InspectedEntry, InspectedJob, InspectedJobEvent, InspectedThread, inspectionSchema, ModelRequest, ThreadState} from '@proxy/contracts/inspect'; - -import type {AuthenticatedRequest} from '../server/middleware/require-auth'; -import {type OrgViewer, requireViewer} from '../utils/user-org'; -import {COMPACT_AT_TOKENS, compactionCut, type SubAgent, toMessages, toTranscript} from '../harness/context'; -import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from '../harness/inference'; -import {requireThread, toPersona} from '../harness/route'; -import type {InspectStore} from './store'; -import {toolsFor} from '../harness/tool-surface'; - -/** - * The inspector's routes, for whoever works on the built-in agent, behind the `debug` experiment - * beside `harness`: the viewer's conversation with a persona as the harness has it, every thread - * of it with its whole log, its jobs and what happened to each, and, for any answer of the model, - * the request it was sent, rebuilt from the log with the functions the harness runs - * (../harness/context.ts) - * and the system prompt the entry kept. The viewer sees their own conversations alone, as on the - * Chat page: a conversation is theirs, whoever they are in the organization. - */ - -function toInspectedEntry(entry: EntryRow): InspectedEntry { - return { - id: entry.id, - seq: entry.seq, - kind: entry.kind, - text: entry.text, - toolCalls: entry.toolCalls, - callId: entry.callId, - isError: entry.isError, - pending: entry.pending, - inputTokens: entry.inputTokens, - outputTokens: entry.outputTokens, - coversSeq: entry.coversSeq, - hasSystem: entry.system !== null, - jobId: entry.jobId, - at: entry.createdAt.toISOString(), - }; -} - -function toInspectedJob(job: JobDetailRow): InspectedJob { - return { - id: job.id, - threadId: job.threadId, - kind: job.kind, - status: job.status, - entryId: job.entryId, - callId: job.callId, - runAt: job.runAt.toISOString(), - claimedBy: job.claimedBy, - leaseExpiresAt: job.leaseExpiresAt?.toISOString() ?? null, - attempts: job.attempts, - error: job.error, - createdAt: job.createdAt.toISOString(), - }; -} - -function toInspectedEvent(event: JobEventRow): InspectedJobEvent { - return {id: event.id, jobId: event.jobId, threadId: event.threadId, kind: event.kind, workerId: event.workerId, error: event.error, at: event.at.toISOString()}; -} - -// Where the thread stands, from the log the model reads: what is pending is not in it yet. -function stateOf(entries: EntryRow[]): ThreadState { - const log = entries.filter((entry) => !entry.pending); - const cut = compactionCut(log); - const lastAnswer = log.findLast((entry) => entry.kind === 'assistant'); - return { - ready: readyForInference(log), - pendingCalls: pendingToolCalls(log), - turnsAlone: turnsAlone(log), - maxTurnsAlone: MAX_TURNS_ALONE, - lastInputTokens: lastAnswer?.inputTokens ?? 0, - compactAtTokens: COMPACT_AT_TOKENS, - compactionCoversSeq: cut === null ? null : (log[cut - 1]?.seq ?? null), - }; -} - -function requireInspector(request: AuthenticatedRequest): Promise> { - return Do(async ($) => { - const viewer = $(await requireViewer(request)); - if (!hasExperiment(viewer, 'harness') || !hasExperiment(viewer, 'debug')) { - return $(Err(ApiErr.forbidden())); - } - return viewer; - }); -} - -// A thread the viewer may inspect: one of their own conversations, or a sub-agent started from one. -function mayInspect(viewer: OrgViewer, thread: ThreadRow): boolean { - return thread.orgId === viewer.orgId && thread.userId === viewer.userId; -} - -function inspectThread(store: InspectStore, thread: ThreadRow, jobs: JobDetailRow[], events: JobEventRow[]): Promise> { - return Do(async ($) => { - const entries = $(await store.listEntriesWithPending(thread.id)); - return { - id: thread.id, - parentThreadId: thread.parentThreadId, - parentCallId: thread.parentCallId, - title: thread.title, - taskId: thread.taskId, - createdAt: thread.createdAt.toISOString(), - entries: entries.map(toInspectedEntry), - jobs: jobs.filter((job) => job.threadId === thread.id).map(toInspectedJob), - events: events.filter((event) => event.threadId === thread.id).map(toInspectedEvent), - state: stateOf(entries), - }; - }); -} - -/** `GET /api/orgs/:org/chat/personas/:personaId/inspect`: the viewer's conversation with the persona as the harness has it, begun where there was none, as the Chat page begins it. */ -function handleInspectThreadRoute(store: InspectStore, request: AuthenticatedRequest): Promise, ApiError>> { - return Do(async ($) => { - const viewer = $(await requireInspector(request)); - const personaId = request.params.personaId ?? ''; - const persona = $(requirePresent($(await store.getPersona(viewer.orgId, personaId)), ApiErr.notFound('persona', personaId))); - const conversation = $(await requireThread(store, viewer, persona)); - const now = new Date(); - const subAgents = $(await store.listSubAgents(conversation.id, now)); - const threads: ThreadRow[] = [conversation, ...subAgents]; - const ids = threads.map((thread) => thread.id); - const [jobs, events] = await Promise.all([store.listThreadJobs(ids), store.listJobEvents(ids)]); - const allJobs = $(jobs); - const allEvents = $(events); - const inspected: InspectedThread[] = []; - for (const thread of threads) { - inspected.push($(await inspectThread(store, thread, allJobs, allEvents))); - } - return {persona: toPersona(viewer, persona), now: now.toISOString(), threads: inspected}; - }); -} - -// The request behind an assistant entry: the log before it, as the model read it, behind the -// system prompt the entry kept. The sub-agents are the thread's as they are now: a message of one -// is led in by its call, which doesn't change. -function turnRequest(thread: ThreadRow, entry: EntryRow, log: EntryRow[], subAgents: SubAgent[]): ModelRequest { - const before = log.filter((candidate) => candidate.seq < entry.seq); - const system: ModelRequest['messages'] = entry.system === null ? [] : [{role: 'system', content: entry.system}]; - return {kind: 'turn', system: entry.system, messages: [...system, ...toMessages(before, subAgents, {clearOld: true})], tools: toolsFor(thread)}; -} - -// The request behind a summary entry: the log it covers, written out for the model to summarize. -function summaryRequest(entry: EntryRow, log: EntryRow[], subAgents: SubAgent[]): ModelRequest { - const covered = log.filter((candidate) => candidate.seq <= (entry.coversSeq ?? -1)); - return { - kind: 'summary', - system: SUMMARIZE, - messages: [ - {role: 'system', content: SUMMARIZE}, - {role: 'user', content: `${toTranscript(covered, subAgents)}\n\n${SUMMARIZE_NOW}`}, - ], - tools: [], - }; -} - -/** `GET /api/orgs/:org/chat/inspect/entries/:entryId/request`: what the model was sent for an assistant or summary entry of one of the viewer's threads, rebuilt from the log; not found for any other entry. */ -function handleInspectRequestRoute(store: InspectStore, request: AuthenticatedRequest): Promise> { - return Do(async ($) => { - const viewer = $(await requireInspector(request)); - const entryId = request.params.entryId ?? ''; - const entry = $(await store.getEntry(entryId)); - const thread = $(requirePresent($(await store.getJobThread(entry.threadId)), ApiErr.notFound('thread', entry.threadId))); - if (!mayInspect(viewer, thread)) { - return $(Err(ApiErr.notFound('entry', entryId))); - } - const log = $(await store.listEntries(thread.id)); - // A conversation's log names its sub-agents; a sub-agent's has none of its own. - const subAgents = thread.parentThreadId === null ? $(await store.listSubAgents(thread.id, new Date())) : []; - if (entry.kind === 'assistant') { - return turnRequest(thread, entry, log, subAgents); - } - if (entry.kind === 'summary') { - return summaryRequest(entry, log, subAgents); - } - return $(Err(ApiErr.notFound('entry', entryId))); - }); -} - -/** The inspector's routes, on the store. */ -export function inspectRoutes(store: InspectStore) { - return { - handleInspectThreadRoute: (request: AuthenticatedRequest) => handleInspectThreadRoute(store, request), - handleInspectRequestRoute: (request: AuthenticatedRequest) => handleInspectRequestRoute(store, request), - }; -} diff --git a/packages/api/src/inspect/store.ts b/packages/api/src/inspect/store.ts deleted file mode 100644 index 2d28d91f..00000000 --- a/packages/api/src/inspect/store.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as harness from '@proxy/db/harness'; -import * as inspect from '@proxy/db/inspect'; - -import type {HarnessStore} from '../harness/store'; -import {harnessStore} from '../harness/store'; - -/** - * What the inspector reads: the harness's own reads it shares (../harness/store), and its own, - * every job of a conversation and a thread's log whole (@proxy/db/inspect). The harness knows - * nothing of it: the inspector reads the harness, never the other way. - */ -export type InspectStore = Pick & { - listJobEvents: typeof harness.listJobEvents; - listThreadJobs: typeof inspect.listThreadJobs; - listEntriesWithPending: typeof inspect.listEntriesWithPending; -}; - -/** The store in the database. */ -export const inspectStore: InspectStore = { - getPersona: harnessStore.getPersona, - getMemberThread: harnessStore.getMemberThread, - ensureMemberThread: harnessStore.ensureMemberThread, - listSubAgents: harnessStore.listSubAgents, - getEntry: harnessStore.getEntry, - getJobThread: harnessStore.getJobThread, - listEntries: harnessStore.listEntries, - listJobEvents: harness.listJobEvents, - listThreadJobs: inspect.listThreadJobs, - listEntriesWithPending: inspect.listEntriesWithPending, -}; diff --git a/packages/api/src/llm/open-router.ts b/packages/api/src/llm/open-router.ts index 6d123a34..a70e94c4 100644 --- a/packages/api/src/llm/open-router.ts +++ b/packages/api/src/llm/open-router.ts @@ -3,6 +3,7 @@ import {z} from 'zod'; import {ApiErr, type ApiError, type ChatModel, Do, type FetchError, httpRequest, parseSchema} from '@proxy/utils'; import type {ToolCall} from '@proxy/utils/harness'; +import type {ModelMessage} from '@proxy/utils/harness-context'; /** * The models this server asks over OpenRouter's chat completions, with the key in @@ -36,14 +37,8 @@ const MAX_SAID = 500; /** A tool as the model is given it: its name, what it does, and the JSON Schema of its arguments. */ export type ModelTool = {name: string; description: string; parameters: object}; -type ModelToolCall = {id: string; type: 'function'; function: {name: string; arguments: string}}; - -/** The conversation as the model reads it, in the chat completions form. */ -export type ModelMessage = - | {role: 'system'; content: string} - | {role: 'user'; content: string} - | {role: 'assistant'; content: string | null; tool_calls?: ModelToolCall[]} - | {role: 'tool'; tool_call_id: string; content: string}; +/** The conversation as the model reads it, in the chat completions form (@proxy/utils/harness-context, which builds it from a log). */ +export type {ModelMessage}; // Which hosts of the model OpenRouter may send the request to. type ProviderPolicy = {order?: string[]; allow_fallbacks?: boolean; data_collection: 'deny'; zdr?: boolean}; diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index 770a5e20..4a7f847f 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -44,8 +44,6 @@ import { import {handleFaviconRoute} from '../favicons/route'; import {chatRoutes} from '../harness/route'; import {harnessStore} from '../harness/store'; -import {inspectRoutes} from '../inspect/route'; -import {inspectStore} from '../inspect/store'; import {handleFillIntentRoute, handlePickIntentRoute} from '../intent/route'; import {handleMcpMethodNotAllowedRoute, handleMcpRoute} from '../mcp/route'; import {handleDeleteMeRoute, handleMeRoute} from '../me/route'; @@ -208,11 +206,6 @@ export function createApiRouter(): express.Router { org.delete('/chat/personas/:personaId', withAuthResult('Delete persona route', chat.handleDeletePersonaRoute)); org.get('/chat/personas/:personaId/thread', withAuthResult('Get chat thread route', chat.handleGetThreadRoute)); org.post('/chat/personas/:personaId/messages', withAuthResult('Post chat message route', chat.handlePostMessageRoute)); - // The inspector, behind the `debug` experiment too: the conversation as the harness has it, - // and what the model was sent for an answer. - const inspect = inspectRoutes(inspectStore); - org.get('/chat/personas/:personaId/inspect', withAuthResult('Inspect chat thread route', inspect.handleInspectThreadRoute)); - org.get('/chat/inspect/entries/:entryId/request', withAuthResult('Inspect chat request route', inspect.handleInspectRequestRoute)); // The agent side: its own sign-in and cookie, and every request made for one of the agent's // tasks, checked against the agent's access and logged with the task. diff --git a/packages/contracts/src/inspect.ts b/packages/contracts/src/inspect.ts deleted file mode 100644 index a3f37fc5..00000000 --- a/packages/contracts/src/inspect.ts +++ /dev/null @@ -1,158 +0,0 @@ -import {z} from 'zod'; - -import {chatPersonaSchema} from './harness'; - -/** - * The inspector (`/api/orgs/:org/chat/…/inspect`), for whoever works on the built-in agent: the - * person's conversation with a persona as the harness has it, every thread of it with its whole - * log, its jobs and what happened to each; and, for an answer of the model, the request it was - * sent. Behind the `debug` experiment beside `harness`. - */ - -const toolCallSchema = z.object({id: z.string(), name: z.string(), arguments: z.string()}); - -export type ToolCall = z.infer; - -export const entryKinds = ['person', 'assistant', 'tool_result', 'event', 'summary'] as const; - -/** An entry of a thread's log, whole: every kind, what is pending too, with what the model's answer cost. */ -export const inspectedEntrySchema = z.object({ - id: z.string(), - seq: z.number(), - kind: z.enum(entryKinds), - text: z.string(), - toolCalls: z.array(toolCallSchema).nullable(), - /** A tool_result's or an event's call, by the id the model gave it. */ - callId: z.string().nullable(), - isError: z.boolean(), - /** Landed while the model was answering: not in the log it reads until the answer is. */ - pending: z.boolean(), - inputTokens: z.number().nullable(), - outputTokens: z.number().nullable(), - /** A summary's reach: the seq of the last entry it stands for. */ - coversSeq: z.number().nullable(), - /** Whether the answer kept the system prompt it was sent, which the request route gives. */ - hasSystem: z.boolean(), - /** The job that wrote the entry; null for what the person wrote. */ - jobId: z.string().nullable(), - at: z.string(), -}); - -export type InspectedEntry = z.infer; - -export const jobStatuses = ['queued', 'running', 'done', 'failed'] as const; - -/** A job as the queue has it, whatever its status. */ -export const inspectedJobSchema = z.object({ - id: z.string(), - threadId: z.string(), - kind: z.enum(['inference', 'tool']), - status: z.enum(jobStatuses), - /** A tool job's call: the entry it is in and the id the model gave it. */ - entryId: z.string().nullable(), - callId: z.string().nullable(), - runAt: z.string(), - /** Which worker holds it, and until when, while it runs. */ - claimedBy: z.string().nullable(), - leaseExpiresAt: z.string().nullable(), - attempts: z.number(), - error: z.string().nullable(), - createdAt: z.string(), -}); - -export type InspectedJob = z.infer; - -/** What happened to a job, one row per happening. */ -export const inspectedJobEventSchema = z.object({ - id: z.string(), - jobId: z.string(), - threadId: z.string(), - kind: z.enum(['claimed', 'requeued', 'done', 'failed']), - workerId: z.string().nullable(), - /** Why, on failed, and on requeued. */ - error: z.string().nullable(), - at: z.string(), -}); - -export type InspectedJobEvent = z.infer; - -/** Where the thread stands for the harness: whether its log is ready for the model and what holds it back, the model's steps alone against the limit, and how close the log is to being summarized. */ -export const threadStateSchema = z.object({ - ready: z.boolean(), - pendingCalls: z.array(toolCallSchema), - turnsAlone: z.number(), - maxTurnsAlone: z.number(), - lastInputTokens: z.number(), - compactAtTokens: z.number(), - /** The seq of the last entry a summary made now would cover; null when the log needs none. */ - compactionCoversSeq: z.number().nullable(), -}); - -export type ThreadState = z.infer; - -/** A thread of the conversation: the conversation itself, or a sub-agent started from it. */ -export const inspectedThreadSchema = z.object({ - id: z.string(), - parentThreadId: z.string().nullable(), - parentCallId: z.string().nullable(), - title: z.string().nullable(), - taskId: z.string().nullable(), - createdAt: z.string(), - entries: z.array(inspectedEntrySchema), - jobs: z.array(inspectedJobSchema), - events: z.array(inspectedJobEventSchema), - state: threadStateSchema, -}); - -export type InspectedThread = z.infer; - -/** `GET chat/personas/:personaId/inspect`: the conversation whole, its thread first, then every sub-agent started from it, oldest first. */ -export const inspectionSchema = z.object({ - persona: chatPersonaSchema, - now: z.string(), - threads: z.array(inspectedThreadSchema), -}); - -export type Inspection = z.infer; - -/** The conversation as the model reads it, in the chat completions form. */ -export const modelMessageSchema = z.discriminatedUnion('role', [ - z.object({role: z.literal('system'), content: z.string()}), - z.object({role: z.literal('user'), content: z.string()}), - z.object({ - role: z.literal('assistant'), - content: z.string().nullable(), - tool_calls: z - .array( - z.object({ - id: z.string(), - type: z.literal('function'), - function: z.object({name: z.string(), arguments: z.string()}), - }), - ) - .optional(), - }), - z.object({role: z.literal('tool'), tool_call_id: z.string(), content: z.string()}), -]); - -export type ModelMessage = z.infer; - -/** A tool as the model is given it: its name, what it does, and the JSON Schema of its arguments. */ -export const modelToolSchema = z.object({ - name: z.string(), - description: z.string(), - parameters: z.unknown(), -}); - -export type ModelTool = z.infer; - -/** `GET chat/inspect/entries/:entryId/request`: what the model was sent for an assistant or summary entry, rebuilt from the log as it was. */ -export const modelRequestSchema = z.object({ - kind: z.enum(['turn', 'summary']), - /** The system prompt as kept; null where the answer was written without the model. */ - system: z.string().nullable(), - messages: z.array(modelMessageSchema), - tools: z.array(modelToolSchema), -}); - -export type ModelRequest = z.infer; diff --git a/packages/db/package.json b/packages/db/package.json index 86a19484..6a75477c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,7 +12,6 @@ "./desktop-auth": "./src/desktop-auth.ts", "./feedback": "./src/feedback.ts", "./harness": "./src/harness.ts", - "./inspect": "./src/inspect.ts", "./vault": "./src/vault.ts", "./invitation": "./src/invitation.ts", "./instance-meta": "./src/instance-meta.ts", diff --git a/packages/db/src/harness.test.ts b/packages/db/src/harness.test.ts index 9c1c9ea1..2e9b8890 100644 --- a/packages/db/src/harness.test.ts +++ b/packages/db/src/harness.test.ts @@ -190,7 +190,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['assistant', job.id, 'You are signed in.', null], ['tool_result', job.id, null, 'c0'], ]); - const events = unwrap(await listJobEvents([thread.id])); + const events = unwrap(await listJobEvents(thread.id)); expect(events.map((event) => [event.jobId, event.kind, event.workerId, event.error])).toEqual([ [job.id, 'claimed', 'worker-a', null], [job.id, 'done', 'worker-a', null], @@ -255,7 +255,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['tool_result', tool.id, false], ['assistant', next.id, true], ]); - const events = unwrap(await listJobEvents([thread.id])); + const events = unwrap(await listJobEvents(thread.id)); expect( events.slice(2).map((event) => [event.jobId, event.kind, event.workerId, event.error]), ).toEqual([ @@ -275,7 +275,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { throw new Error('nothing to claim'); } unwrap(await finishJob(job.id, 'worker-e')); - const events = unwrap(await listJobEvents([thread.id])); + const events = unwrap(await listJobEvents(thread.id)); expect(events.slice(-2).map((event) => [event.jobId, event.kind, event.workerId])).toEqual([ [job.id, 'claimed', 'worker-e'], [job.id, 'done', 'worker-e'], diff --git a/packages/db/src/harness.ts b/packages/db/src/harness.ts index 500f54ea..0a5d1a1f 100644 --- a/packages/db/src/harness.ts +++ b/packages/db/src/harness.ts @@ -211,7 +211,7 @@ function toPersona(persona: { return {...persona, look: parseLook(persona.look)}; } -/** The columns of an entry, as `EntryRow` has them; the inspector reads with it too. */ +/** The columns of an entry, as `EntryRow` has them; the admin reads with it too. */ export const entrySelect = { id: true, seq: true, @@ -230,8 +230,7 @@ export const entrySelect = { createdAt: true, } as const; -/** The columns of a job the worker reads; the inspector reads more. */ -export const jobSelect = { +const jobSelect = { id: true, threadId: true, kind: true, @@ -541,11 +540,11 @@ export async function listSubAgents( ); } -/** What happened to the threads' jobs, in order: every claim, requeue, finish and failure. */ -export async function listJobEvents(threadIds: string[]): Promise> { +/** What happened to the thread's jobs, in order: every claim, requeue, finish and failure. */ +export async function listJobEvents(threadId: string): Promise> { return wrapDb(() => db.agentJobEvent.findMany({ - where: {threadId: {in: threadIds}}, + where: {threadId}, select: jobEventSelect, orderBy: {at: 'asc'}, }), diff --git a/packages/db/src/inspect.ts b/packages/db/src/inspect.ts deleted file mode 100644 index 056c6bff..00000000 --- a/packages/db/src/inspect.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type {Result} from 'ts-results-es'; - -import {type ApiError, wrapDb} from '@proxy/utils'; - -import {db} from '.'; -import {type EntryRow, entrySelect, type JobRow, jobSelect, toEntry} from './harness'; - -/** - * What the inspector reads of the built-in agent beyond what the harness does (./harness): every - * job of a conversation whatever its status, and a thread's log with what is pending. Reads - * only; the harness alone writes. - */ - -/** A job with its whole record: beside what the worker reads, why it failed, its lease and when it was made. */ -export type JobDetailRow = JobRow & { - leaseExpiresAt: Date | null; - error: string | null; - createdAt: Date; -}; - -const jobDetailSelect = {...jobSelect, leaseExpiresAt: true, error: true, createdAt: true} as const; - -/** Every job of the threads, whatever its status, oldest first: the queue and its history. */ -export async function listThreadJobs( - threadIds: string[], -): Promise> { - return wrapDb(() => - db.agentJob.findMany({ - where: {threadId: {in: threadIds}}, - select: jobDetailSelect, - orderBy: {createdAt: 'asc'}, - }), - ); -} - -/** The thread's whole log: what the model reads in order, then what is pending, in the order it landed. */ -export async function listEntriesWithPending( - threadId: string, -): Promise> { - return ( - await wrapDb(() => - db.agentEntry.findMany({ - where: {threadId}, - select: entrySelect, - orderBy: [{pending: 'asc'}, {seq: 'asc'}], - }), - ) - ).map((entries) => entries.map(toEntry)); -} diff --git a/packages/utils/package.json b/packages/utils/package.json index 71a5121a..b5b024ad 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -9,6 +9,7 @@ "./days": "./src/days.ts", "./email-providers": "./src/email-providers.ts", "./harness": "./src/harness.ts", + "./harness-context": "./src/harness-context.ts", "./secret-crypto": "./src/secret-crypto.ts", "./tasks": "./src/tasks.ts", "./units": "./src/units.ts", diff --git a/packages/utils/src/experiments.ts b/packages/utils/src/experiments.ts index 37bf53f3..2d2928fa 100644 --- a/packages/utils/src/experiments.ts +++ b/packages/utils/src/experiments.ts @@ -9,9 +9,6 @@ export const EXPERIMENTS = { // The built-in agent: a conversation with an agent Personal Agent Proxy runs itself, behind its // own gate, on the Chat page. harness: 'Built-in agent', - // Debugging: the inspector of the built-in agent, for whoever works on it, every thread of a - // conversation with its whole log, its jobs and what the model was sent, on its own page. - debug: 'Debugging', } as const satisfies Record; export type Experiment = keyof typeof EXPERIMENTS; diff --git a/packages/api/src/harness/context.test.ts b/packages/utils/src/harness-context.test.ts similarity index 65% rename from packages/api/src/harness/context.test.ts rename to packages/utils/src/harness-context.test.ts index b59eab12..7d142296 100644 --- a/packages/api/src/harness/context.test.ts +++ b/packages/utils/src/harness-context.test.ts @@ -1,35 +1,37 @@ import {describe, expect, test} from 'bun:test'; -import type {EntryRow} from '@proxy/db/harness'; +import { + COMPACT_AT_TOKENS, + compactionCut, + type ContextEntry, + toMessages, + toTranscript, +} from './harness-context'; -import {COMPACT_AT_TOKENS, compactionCut, toMessages, toTranscript} from './context'; +type Kind = ContextEntry['kind']; -const now = new Date('2026-10-21T10:00:00Z'); - -type Kind = EntryRow['kind']; - -function entry(seq: number, kind: Kind, text: string, extra: Partial> = {}): EntryRow { +function entry( + seq: number, + kind: Kind, + text: string, + extra: Partial> = {}, +): ContextEntry { return { - id: `e${seq}`, seq, - threadId: 'thread-1', kind, - pending: false, text, toolCalls: extra.toolCalls ?? null, callId: extra.callId ?? null, isError: false, inputTokens: extra.inputTokens ?? null, - outputTokens: null, coversSeq: extra.coversSeq ?? null, - system: null, - jobId: null, - createdAt: now, }; } -const read = (seq: number, id: string, inputTokens = 10) => entry(seq, 'assistant', '', {toolCalls: [{id, name: 'read', arguments: '{}'}], inputTokens}); -const result = (seq: number, id: string, text = 'rows') => entry(seq, 'tool_result', text, {callId: id}); +const read = (seq: number, id: string, inputTokens = 10) => + entry(seq, 'assistant', '', {toolCalls: [{id, name: 'read', arguments: '{}'}], inputTokens}); +const result = (seq: number, id: string, text = 'rows') => + entry(seq, 'tool_result', text, {callId: id}); // Three turns of the person, each answered after a read. const long = [ @@ -50,7 +52,11 @@ describe('toMessages', () => { test('gives the log in the chat form, as it is, where it has no summary', () => { expect(toMessages(long.slice(0, 4), [], {clearOld: true})).toEqual([ {role: 'user', content: 'One'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c1', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c1', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c1', content: 'rows one'}, {role: 'assistant', content: 'Done one'}, ]); @@ -61,33 +67,72 @@ describe('toMessages', () => { const messages = toMessages(log, [], {clearOld: true}); - expect(messages[0]).toEqual({role: 'user', content: expect.stringContaining('The person asked for one.')}); + expect(messages[0]).toEqual({ + role: 'user', + content: expect.stringContaining('The person asked for one.'), + }); expect(String(messages[0]?.content)).toContain('The person did not write this'); expect(messages.slice(1)).toEqual([ {role: 'user', content: 'Two'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c2', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c2', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c2', content: expect.stringContaining('cleared')}, {role: 'assistant', content: 'Done two'}, {role: 'user', content: 'Three'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c3', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c3', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c3', content: 'rows three'}, ]); - expect(toMessages(log, [], {clearOld: false})[3]).toEqual({role: 'tool', tool_call_id: 'c2', content: 'rows two'}); + expect(toMessages(log, [], {clearOld: false})[3]).toEqual({ + role: 'tool', + tool_call_id: 'c2', + content: 'rows two', + }); }); test('reads what a sub-agent said as a message, led in with whose it is, by id, and clears old ones too', () => { const delegated = entry(1, 'person', 'Find my invoices'); - const call = entry(2, 'assistant', 'Looking.', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}'}], inputTokens: 10}); + const call = entry(2, 'assistant', 'Looking.', { + toolCalls: [ + { + id: 'd1', + name: 'create_subagent', + arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}', + }, + ], + inputTokens: 10, + }); const started = entry(3, 'tool_result', 'Started, id sub-1.', {callId: 'd1'}); const report = entry(4, 'event', 'Three invoices from Dana.', {callId: 'd1'}); - const failed = {...entry(5, 'event', 'The sub-agent failed: boom', {callId: 'd1'}), isError: true}; + const failed = { + ...entry(5, 'event', 'The sub-agent failed: boom', {callId: 'd1'}), + isError: true, + }; const subAgents = [{id: 'sub-1', parentCallId: 'd1', title: 'Find the invoices'}]; - const messages = toMessages([delegated, call, started, report, failed], subAgents, {clearOld: true}); + const messages = toMessages([delegated, call, started, report, failed], subAgents, { + clearOld: true, + }); - expect(messages[3]).toEqual({role: 'user', content: expect.stringContaining('A message from the sub-agent “Find the invoices” (id sub-1)')}); + expect(messages[3]).toEqual({ + role: 'user', + content: expect.stringContaining( + 'A message from the sub-agent “Find the invoices” (id sub-1)', + ), + }); expect(String(messages[3]?.content)).toContain('Three invoices from Dana.'); - expect(messages[4]).toEqual({role: 'user', content: expect.stringContaining('What the sub-agent “Find the invoices” (id sub-1) failed with')}); + expect(messages[4]).toEqual({ + role: 'user', + content: expect.stringContaining( + 'What the sub-agent “Find the invoices” (id sub-1) failed with', + ), + }); const later = [ delegated, @@ -104,13 +149,32 @@ describe('toMessages', () => { }); test('leads a message in by the call where no sub-agent was started by it', () => { - const call = entry(2, 'assistant', '', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":'}], inputTokens: 10}); - const messages = toMessages([entry(1, 'person', 'Go'), call, entry(3, 'tool_result', 'Started.', {callId: 'd1'}), entry(4, 'event', 'Done.', {callId: 'd1'})], [], {clearOld: true}); - expect(messages[3]).toEqual({role: 'user', content: expect.stringContaining('A message from your call d1')}); + const call = entry(2, 'assistant', '', { + toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":'}], + inputTokens: 10, + }); + const messages = toMessages( + [ + entry(1, 'person', 'Go'), + call, + entry(3, 'tool_result', 'Started.', {callId: 'd1'}), + entry(4, 'event', 'Done.', {callId: 'd1'}), + ], + [], + {clearOld: true}, + ); + expect(messages[3]).toEqual({ + role: 'user', + content: expect.stringContaining('A message from your call d1'), + }); }); test('with summaries on summaries, only the last counts', () => { - const log = [...long, entry(12, 'summary', 'First.', {coversSeq: 4}), entry(13, 'summary', 'Second.', {coversSeq: 8})]; + const log = [ + ...long, + entry(12, 'summary', 'First.', {coversSeq: 4}), + entry(13, 'summary', 'Second.', {coversSeq: 8}), + ]; const messages = toMessages(log, [], {clearOld: true}); @@ -130,7 +194,9 @@ describe('compactionCut', () => { }); test('is nothing where the last turns are the whole log', () => { - expect(compactionCut([entry(1, 'person', 'One'), read(2, 'c1', COMPACT_AT_TOKENS), result(3, 'c1')])).toBeNull(); + expect( + compactionCut([entry(1, 'person', 'One'), read(2, 'c1', COMPACT_AT_TOKENS), result(3, 'c1')]), + ).toBeNull(); }); test('moves back so that no call is left without its result', () => { @@ -178,7 +244,12 @@ describe('compactionCut', () => { ]; expect(compactionCut(log)).toBe(3); - const summarized = [...log, entry(10, 'summary', 'Two reads so far.', {coversSeq: 3}), read(11, 'c5', COMPACT_AT_TOKENS), result(12, 'c5')]; + const summarized = [ + ...log, + entry(10, 'summary', 'Two reads so far.', {coversSeq: 3}), + read(11, 'c5', COMPACT_AT_TOKENS), + result(12, 'c5'), + ]; expect(compactionCut(summarized)).toBe(5); }); }); @@ -186,7 +257,16 @@ describe('compactionCut', () => { describe('toTranscript', () => { test('writes the log out as text from its last summary on, the calls, their answers and the sub-agents’ messages with whose they are', () => { const delegated = entry(1, 'person', 'Find my invoices'); - const call = entry(2, 'assistant', 'Looking.', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}'}], inputTokens: 10}); + const call = entry(2, 'assistant', 'Looking.', { + toolCalls: [ + { + id: 'd1', + name: 'create_subagent', + arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}', + }, + ], + inputTokens: 10, + }); const started = entry(3, 'tool_result', 'Started, id sub-1.', {callId: 'd1'}); const report = entry(4, 'event', 'Three invoices from Dana.', {callId: 'd1'}); const failed = {...entry(5, 'tool_result', 'No such tool.', {callId: 'x9'}), isError: true}; @@ -202,7 +282,10 @@ describe('toTranscript', () => { ].join('\n\n'), ); - const transcript = toTranscript([...long, entry(12, 'summary', 'The person asked for one.', {coversSeq: 4})], []); + const transcript = toTranscript( + [...long, entry(12, 'summary', 'The person asked for one.', {coversSeq: 4})], + [], + ); expect(transcript.startsWith('What came before in this conversation')).toBe(true); expect(transcript).toContain('The person asked for one.\n\nPerson:\nTwo'); expect(transcript).not.toContain('rows one'); diff --git a/packages/api/src/harness/context.ts b/packages/utils/src/harness-context.ts similarity index 62% rename from packages/api/src/harness/context.ts rename to packages/utils/src/harness-context.ts index c488bf1e..ca7b8808 100644 --- a/packages/api/src/harness/context.ts +++ b/packages/utils/src/harness-context.ts @@ -1,10 +1,35 @@ -import {pendingToolCalls, type ToolCall} from '@proxy/utils/harness'; -import type {EntryRow} from '@proxy/db/harness'; - -import type {ModelMessage} from '../llm/open-router'; +import {pendingToolCalls, type ToolCall} from './harness'; + +/** The conversation as the model reads it, in the chat completions form. */ +export type ModelMessage = + | {role: 'system'; content: string} + | {role: 'user'; content: string} + | { + role: 'assistant'; + content: string | null; + tool_calls?: Array<{ + id: string; + type: 'function'; + function: {name: string; arguments: string}; + }>; + } + | {role: 'tool'; tool_call_id: string; content: string}; + +/** An entry of the log, as much of it as reading the log needs: its kind and text, its calls or the call it answers, its place, how far a summary reaches, and what an answer cost. */ +export type ContextEntry = { + kind: 'person' | 'assistant' | 'tool_result' | 'event' | 'summary'; + text: string; + toolCalls: ToolCall[] | null; + callId: string | null; + isError: boolean; + seq: number; + coversSeq: number | null; + inputTokens: number | null; +}; /** - * The log as the model reads it. An event, what a sub-agent said to the agent that started it, is + * The log as the model reads it, here rather than in the api so that the admin (the internal + * repo) can rebuild what the model was sent from a log it reads. An event, what a sub-agent said to the agent that started it, is * read as a message, since it came on its own, after the call that started the sub-agent was * answered with its id; which sub-agent, by that call, is known from the thread's sub-agents * (`SubAgent`), given alongside the log. A log that has grown long @@ -26,7 +51,8 @@ const KEEP_TURNS = 2; // person than that: a sub-agent's, one brief and the calls made on it. const KEEP_STEPS = 3; -const SUMMARY_LEAD = 'What came before in this conversation, summarized so that it fits. The person did not write this; take it as what happened.'; +const SUMMARY_LEAD = + 'What came before in this conversation, summarized so that it fits. The person did not write this; take it as what happened.'; const CLEARED = '(This was cleared from the conversation to save room. Ask again if you need it.)'; @@ -34,21 +60,25 @@ const CLEARED = '(This was cleared from the conversation to save room. Ask again export type SubAgent = {id: string; parentCallId: string | null; title: string | null}; // The call an entry answers, by the id the model gave it, found in the log. -function callOf(entries: EntryRow[], callId: string | null): ToolCall | undefined { - return entries.flatMap((entry) => entry.toolCalls ?? []).find((candidate) => candidate.id === callId); +function callOf(entries: ContextEntry[], callId: string | null): ToolCall | undefined { + return entries + .flatMap((entry) => entry.toolCalls ?? []) + .find((candidate) => candidate.id === callId); } // How an event is led in: which sub-agent said it, by the call that started it. -function eventLead(subAgents: SubAgent[], event: EntryRow): string { +function eventLead(subAgents: SubAgent[], event: ContextEntry): string { const from = subAgents.find((candidate) => candidate.parentCallId === event.callId); - const who = from ? `the sub-agent “${from.title ?? ''}” (id ${from.id})` : `your call ${event.callId ?? ''}`; + const who = from + ? `the sub-agent “${from.title ?? ''}” (id ${from.id})` + : `your call ${event.callId ?? ''}`; if (event.isError) { return `What ${who} failed with. The person did not write this.`; } return `A message from ${who}. The person did not write this; it is for you to act on or tell them of.`; } -function toMessage(subAgents: SubAgent[], entry: EntryRow): ModelMessage { +function toMessage(subAgents: SubAgent[], entry: ContextEntry): ModelMessage { if (entry.kind === 'person') { return {role: 'user', content: entry.text}; } @@ -65,16 +95,30 @@ function toMessage(subAgents: SubAgent[], entry: EntryRow): ModelMessage { return { role: 'assistant', content: entry.text === '' && calls.length > 0 ? null : entry.text, - ...(calls.length > 0 ? {tool_calls: calls.map((call) => ({id: call.id, type: 'function' as const, function: {name: call.name, arguments: call.arguments}}))} : {}), + ...(calls.length > 0 + ? { + tool_calls: calls.map((call) => ({ + id: call.id, + type: 'function' as const, + function: {name: call.name, arguments: call.arguments}, + })), + } + : {}), }; } // The log from its last summary on: the summary, then the entries it doesn't cover, no other // summary among them. -function sinceSummary(entries: EntryRow[]): {summary: EntryRow | null; tail: EntryRow[]} { +function sinceSummary(entries: ContextEntry[]): { + summary: ContextEntry | null; + tail: ContextEntry[]; +} { const summary = entries.findLast((entry) => entry.kind === 'summary') ?? null; const covered = summary?.coversSeq ?? -1; - return {summary, tail: entries.filter((entry) => entry.kind !== 'summary' && entry.seq > covered)}; + return { + summary, + tail: entries.filter((entry) => entry.kind !== 'summary' && entry.seq > covered), + }; } /** @@ -82,18 +126,29 @@ function sinceSummary(entries: EntryRow[]): {summary: EntryRow | null; tail: Ent * what events brought before the person's current turn is cleared, once there is a summary: that * is how the model reads a log, where a summary is made from the log as it is. */ -export function toMessages(entries: EntryRow[], subAgents: SubAgent[], {clearOld}: {clearOld: boolean}): ModelMessage[] { +export function toMessages( + entries: ContextEntry[], + subAgents: SubAgent[], + {clearOld}: {clearOld: boolean}, +): ModelMessage[] { const {summary, tail} = sinceSummary(entries); if (!summary) { return tail.map((entry) => toMessage(subAgents, entry)); } const current = clearOld ? tail.findLastIndex((entry) => entry.kind === 'person') : -1; - const bulky = (entry: EntryRow) => entry.kind === 'tool_result' || entry.kind === 'event'; - return [toMessage(subAgents, summary), ...tail.map((entry, index) => (bulky(entry) && index < current ? toMessage(subAgents, {...entry, text: CLEARED}) : toMessage(subAgents, entry)))]; + const bulky = (entry: ContextEntry) => entry.kind === 'tool_result' || entry.kind === 'event'; + return [ + toMessage(subAgents, summary), + ...tail.map((entry, index) => + bulky(entry) && index < current + ? toMessage(subAgents, {...entry, text: CLEARED}) + : toMessage(subAgents, entry), + ), + ]; } // An entry as the transcript has it. -function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): string { +function toLine(entries: ContextEntry[], subAgents: SubAgent[], entry: ContextEntry): string { if (entry.kind === 'person') { return `Person:\n${entry.text}`; } @@ -106,7 +161,9 @@ function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): st if (entry.kind === 'summary') { return `${SUMMARY_LEAD}\n${entry.text}`; } - const calls = (entry.toolCalls ?? []).map((call) => `Agent called ${call.name} with ${call.arguments}`); + const calls = (entry.toolCalls ?? []).map( + (call) => `Agent called ${call.name} with ${call.arguments}`, + ); return [...(entry.text === '' ? [] : [`Agent:\n${entry.text}`]), ...calls].join('\n'); } @@ -115,9 +172,11 @@ function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): st * what they answered written out, since a request that carries them as such has to carry the * tools too, which a summary needs none of. */ -export function toTranscript(entries: EntryRow[], subAgents: SubAgent[]): string { +export function toTranscript(entries: ContextEntry[], subAgents: SubAgent[]): string { const {summary, tail} = sinceSummary(entries); - return [...(summary ? [summary] : []), ...tail].map((entry) => toLine(entries, subAgents, entry)).join('\n\n'); + return [...(summary ? [summary] : []), ...tail] + .map((entry) => toLine(entries, subAgents, entry)) + .join('\n\n'); } /** @@ -126,7 +185,7 @@ export function toTranscript(entries: EntryRow[], subAgents: SubAgent[]): string * steps, cut so that no call is left without its result. Null when the log needs none: the last * request to the model was under the limit, or nothing since the last summary would be covered. */ -export function compactionCut(entries: EntryRow[]): number | null { +export function compactionCut(entries: ContextEntry[]): number | null { const {summary, tail} = sinceSummary(entries); const lastAnswer = tail.findLast((entry) => entry.kind === 'assistant'); if ((lastAnswer?.inputTokens ?? 0) < COMPACT_AT_TOKENS) { diff --git a/packages/web/src/inspect/client.ts b/packages/web/src/inspect/client.ts deleted file mode 100644 index f78d8b86..00000000 --- a/packages/web/src/inspect/client.ts +++ /dev/null @@ -1,17 +0,0 @@ -import {inspectionSchema, modelRequestSchema} from '@proxy/contracts/inspect'; -import {apiRequest, orgApi} from '@/client/request'; - -// The inspector's api (@proxy/contracts/inspect): the person's conversation with a persona as the -// harness has it, and, for an answer of the model, the request it was sent. - -export function getInspection(org: string, personaId: string) { - return apiRequest('GET', `${orgApi(org)}/chat/personas/${personaId}/inspect`, inspectionSchema); -} - -export function getRequest(org: string, entryId: string) { - return apiRequest( - 'GET', - `${orgApi(org)}/chat/inspect/entries/${entryId}/request`, - modelRequestSchema, - ); -} diff --git a/packages/web/src/inspect/derive.test.ts b/packages/web/src/inspect/derive.test.ts deleted file mode 100644 index 42a83483..00000000 --- a/packages/web/src/inspect/derive.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -import {describe, expect, test} from 'bun:test'; -import type {InspectedEntry, InspectedJob, InspectedThread} from '@proxy/contracts/inspect'; -import { - coveredSeq, - formatDue, - formatDuration, - jobTitle, - lastAttempt, - threadStatus, - tokensOf, -} from './derive'; - -const now = '2026-10-21T10:00:00.000Z'; - -function job(extra: Partial): InspectedJob { - return { - id: 'job-1', - threadId: 'thread-1', - kind: 'inference', - status: 'queued', - entryId: null, - callId: null, - runAt: now, - claimedBy: null, - leaseExpiresAt: null, - attempts: 0, - error: null, - createdAt: now, - ...extra, - }; -} - -function entry(extra: Partial): InspectedEntry { - return { - id: 'e1', - seq: 1, - kind: 'person', - text: '', - toolCalls: null, - callId: null, - isError: false, - pending: false, - inputTokens: null, - outputTokens: null, - coversSeq: null, - hasSystem: false, - jobId: null, - at: now, - ...extra, - }; -} - -function thread(extra: Partial): InspectedThread { - return { - id: 'thread-1', - parentThreadId: null, - parentCallId: null, - title: null, - taskId: null, - createdAt: now, - entries: [], - jobs: [], - events: [], - state: { - ready: false, - pendingCalls: [], - turnsAlone: 0, - maxTurnsAlone: 30, - lastInputTokens: 0, - compactAtTokens: 80_000, - compactionCoversSeq: null, - }, - ...extra, - }; -} - -describe('threadStatus', () => { - test('is at work while a job runs, and queued while one is due', () => { - expect(threadStatus(thread({jobs: [job({status: 'running'})]}), now).kind).toBe('working'); - expect(threadStatus(thread({jobs: [job({status: 'queued'})]}), now).kind).toBe('queued'); - }); - - test('is asleep until the first of its jobs due later, and only when every open job is', () => { - const later = job({id: 'j2', kind: 'tool', runAt: '2026-10-21T10:05:00.000Z'}); - const sooner = job({id: 'j3', kind: 'tool', runAt: '2026-10-21T10:01:00.000Z'}); - expect(threadStatus(thread({jobs: [later, sooner]}), now)).toEqual({ - kind: 'asleep', - until: '2026-10-21T10:01:00.000Z', - }); - expect(threadStatus(thread({jobs: [later, job({status: 'queued'})]}), now).kind).toBe('queued'); - }); - - test('waits for the person with nothing open, and a sub-agent with nothing open has stopped, or failed where its last job did', () => { - expect(threadStatus(thread({jobs: [job({status: 'done'})]}), now).kind).toBe('waiting'); - expect( - threadStatus(thread({parentThreadId: 'thread-0', jobs: [job({status: 'done'})]}), now).kind, - ).toBe('stopped'); - expect( - threadStatus(thread({parentThreadId: 'thread-0', jobs: [job({status: 'failed'})]}), now).kind, - ).toBe('failed'); - }); -}); - -describe('jobTitle', () => { - test('names a tool job by its call, found in the entry it is in', () => { - const entries = [ - entry({id: 'e2', kind: 'assistant', toolCalls: [{id: 'c1', name: 'read', arguments: '{}'}]}), - ]; - expect(jobTitle(job({kind: 'tool', entryId: 'e2', callId: 'c1'}), entries)).toBe('read'); - expect(jobTitle(job({kind: 'tool', entryId: 'e9', callId: 'c1'}), entries)).toBe('tool'); - expect(jobTitle(job({}), entries)).toBe('inference'); - }); -}); - -describe('the log’s sums', () => { - test('add up what the answers cost, and say how far the latest summary reaches', () => { - const entries = [ - entry({id: 'e1', kind: 'assistant', inputTokens: 100, outputTokens: 10}), - entry({id: 'e2', seq: 2, kind: 'summary', coversSeq: 1, inputTokens: 50, outputTokens: 5}), - entry({id: 'e3', seq: 3, kind: 'assistant', inputTokens: 20, outputTokens: 2}), - ]; - expect(tokensOf(entries)).toEqual({input: 170, output: 17}); - expect(coveredSeq(entries)).toBe(1); - expect(coveredSeq([])).toBe(-1); - }); -}); - -describe('the times', () => { - test('are formatted to what matters at their size', () => { - expect(formatDuration(340)).toBe('340ms'); - expect(formatDuration(1234)).toBe('1.2s'); - expect(formatDuration(130_000)).toBe('2m 10s'); - expect(formatDue('2026-10-21T10:00:12.000Z', now)).toBe('in 12.0s'); - expect(formatDue('2026-10-21T09:00:00.000Z', now)).toBe('due'); - }); - - test('give a job’s last attempt from its events: the claim, and how long it ran once it ended', () => { - const events = [ - { - id: 'ev1', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'claimed' as const, - workerId: 'w1', - error: null, - at: '2026-10-21T10:00:00.000Z', - }, - { - id: 'ev2', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'requeued' as const, - workerId: 'w1', - error: 'down', - at: '2026-10-21T10:00:01.000Z', - }, - { - id: 'ev3', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'claimed' as const, - workerId: 'w2', - error: null, - at: '2026-10-21T10:00:06.000Z', - }, - ]; - expect(lastAttempt(job({}), events)).toEqual({ - claimedAt: '2026-10-21T10:00:06.000Z', - ranMs: null, - }); - const done = { - id: 'ev4', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'done' as const, - workerId: 'w2', - error: null, - at: '2026-10-21T10:00:08.500Z', - }; - expect(lastAttempt(job({}), [...events, done])).toEqual({ - claimedAt: '2026-10-21T10:00:06.000Z', - ranMs: 2500, - }); - expect(lastAttempt(job({}), [])).toEqual({claimedAt: null, ranMs: null}); - }); -}); diff --git a/packages/web/src/inspect/derive.ts b/packages/web/src/inspect/derive.ts deleted file mode 100644 index 162a106f..00000000 --- a/packages/web/src/inspect/derive.ts +++ /dev/null @@ -1,144 +0,0 @@ -import type { - InspectedEntry, - InspectedJob, - InspectedJobEvent, - InspectedThread, -} from '@proxy/contracts/inspect'; - -/** - * What the inspector says of a thread, a job and a log, worked out from what the api gives: - * where a thread stands, what a job is for and how long it took, what a summary covers. - */ - -/** Where a thread stands, in a word: at work, queued or asleep on a job; with nothing open, a conversation waits for the person, and a sub-agent has stopped, or failed where its last job did. */ -export type ThreadStatus = { - kind: 'working' | 'queued' | 'asleep' | 'waiting' | 'stopped' | 'failed'; - // Until when, for one asleep. - until: string | null; -}; - -const SLEEP_SOON_MS = 1000; - -export function threadStatus(thread: InspectedThread, now: string): ThreadStatus { - const open = thread.jobs.filter((job) => job.status === 'queued' || job.status === 'running'); - if (open.some((job) => job.status === 'running')) { - return {kind: 'working', until: null}; - } - if (open.length === 0) { - if (thread.parentThreadId === null) { - return {kind: 'waiting', until: null}; - } - const last = thread.jobs.at(-1); - return {kind: last?.status === 'failed' ? 'failed' : 'stopped', until: null}; - } - const time = new Date(now).getTime(); - // A queued job due later is a sleep; one due now is about to be claimed. - const later = open.filter((job) => new Date(job.runAt).getTime() - time > SLEEP_SOON_MS); - if (later.length === open.length) { - const first = [...later].sort((a, b) => a.runAt.localeCompare(b.runAt))[0]; - return {kind: 'asleep', until: first?.runAt ?? null}; - } - return {kind: 'queued', until: null}; -} - -/** What a thread is called in a list: the conversation, or a sub-agent by its title. */ -export function threadLabel(thread: InspectedThread): string { - return thread.parentThreadId === null ? 'Conversation' : (thread.title ?? 'Sub-agent'); -} - -/** Each of a job's events with how long after the attempt's claim it came; null for a claim, and before any. */ -export function timedEvents( - events: InspectedJobEvent[], -): Array<{event: InspectedJobEvent; sinceClaimMs: number | null}> { - let claimedAt: string | null = null; - return events.map((event) => { - const since = - event.kind !== 'claimed' && claimedAt !== null - ? new Date(event.at).getTime() - new Date(claimedAt).getTime() - : null; - if (event.kind === 'claimed') { - claimedAt = event.at; - } - return {event, sinceClaimMs: since}; - }); -} - -/** What a job is for: the model's turn, or the call it runs, by name, found in the entry the call is in. */ -export function jobTitle(job: InspectedJob, entries: InspectedEntry[]): string { - if (job.kind === 'inference') { - return 'inference'; - } - const entry = entries.find((candidate) => candidate.id === job.entryId); - const call = entry?.toolCalls?.find((candidate) => candidate.id === job.callId); - return call ? call.name : 'tool'; -} - -/** The sum of what the model's answers in the log cost. */ -export function tokensOf(entries: InspectedEntry[]): {input: number; output: number} { - return entries.reduce( - (sum, entry) => ({ - input: sum.input + (entry.inputTokens ?? 0), - output: sum.output + (entry.outputTokens ?? 0), - }), - {input: 0, output: 0}, - ); -} - -/** The seq of the last entry the log's latest summary stands for; -1 where there is none. */ -export function coveredSeq(entries: InspectedEntry[]): number { - return entries.findLast((entry) => entry.kind === 'summary')?.coversSeq ?? -1; -} - -/** "340ms", "1.2s", "2m 10s". */ -export function formatDuration(ms: number): string { - if (ms < 1000) { - return `${Math.max(0, Math.round(ms))}ms`; - } - if (ms < 60_000) { - return `${(ms / 1000).toFixed(1)}s`; - } - const minutes = Math.floor(ms / 60_000); - const seconds = Math.round((ms % 60_000) / 1000); - return `${minutes}m ${seconds}s`; -} - -/** "in 12s", "in 3m 0s"; "due" once the time has come. */ -export function formatDue(runAt: string, now: string): string { - const ms = new Date(runAt).getTime() - new Date(now).getTime(); - if (ms <= 0) { - return 'due'; - } - return `in ${formatDuration(ms)}`; -} - -/** The job's last attempt, from its events: when it was claimed, and how long it ran, where it ended. */ -export function lastAttempt( - job: InspectedJob, - events: InspectedJobEvent[], -): {claimedAt: string | null; ranMs: number | null} { - const own = events.filter((event) => event.jobId === job.id); - const claimed = own.findLast((event) => event.kind === 'claimed') ?? null; - const ended = - own.findLast( - (event) => event.kind !== 'claimed' && (claimed === null || event.at >= claimed.at), - ) ?? null; - if (!claimed) { - return {claimedAt: null, ranMs: null}; - } - if (!ended) { - return {claimedAt: claimed.at, ranMs: null}; - } - return { - claimedAt: claimed.at, - ranMs: new Date(ended.at).getTime() - new Date(claimed.at).getTime(), - }; -} - -/** The arguments pretty-printed where they are JSON, as written otherwise. */ -export function prettyJson(text: string): string { - try { - return JSON.stringify(JSON.parse(text), null, 2); - } catch { - return text; - } -} diff --git a/packages/web/src/inspect/inspect-page.test.tsx b/packages/web/src/inspect/inspect-page.test.tsx deleted file mode 100644 index 5449bcb6..00000000 --- a/packages/web/src/inspect/inspect-page.test.tsx +++ /dev/null @@ -1,327 +0,0 @@ -/// - -import {afterEach, beforeEach, describe, expect, mock, test} from 'bun:test'; -import {cleanup, fireEvent, render, screen, waitFor, within} from '@testing-library/react'; -import {createMemoryRouter, RouterProvider} from 'react-router'; -import {TooltipProvider} from '@proxy/ui/components/tooltip'; -import {ThemeProvider} from '@proxy/ui/components/theme-provider'; -import {HumanSide} from '@/components/human-side'; -import type {ChatPersona} from '@proxy/contracts/harness'; -import type { - InspectedEntry, - InspectedJob, - InspectedThread, - Inspection, - ModelRequest, -} from '@proxy/contracts/inspect'; -import {InspectPage} from '@/inspect/inspect-page'; -import {testHuman} from '@/test-org'; - -const now = '2026-10-21T10:00:00.000Z'; -const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; -const persona: ChatPersona = { - id: 'persona-1', - name: 'Finance', - look, - color: 'blue', - mayChange: true, - lastMessage: null, -}; - -function entry( - seq: number, - kind: InspectedEntry['kind'], - text: string, - extra: Partial = {}, -): InspectedEntry { - return { - id: `e${seq}`, - seq, - kind, - text, - toolCalls: null, - callId: null, - isError: false, - pending: false, - inputTokens: null, - outputTokens: null, - coversSeq: null, - hasSystem: false, - jobId: null, - at: now, - ...extra, - }; -} - -function job(id: string, threadId: string, extra: Partial = {}): InspectedJob { - return { - id, - threadId, - kind: 'inference', - status: 'done', - entryId: null, - callId: null, - runAt: now, - claimedBy: null, - leaseExpiresAt: null, - attempts: 1, - error: null, - createdAt: now, - ...extra, - }; -} - -const state = { - ready: false, - pendingCalls: [], - turnsAlone: 1, - maxTurnsAlone: 30, - lastInputTokens: 1200, - compactAtTokens: 80_000, - compactionCoversSeq: null, -}; - -// The person asked for invoices, the agent delegated, and the sub-agent is reading. -const conversation: InspectedThread = { - id: 'thread-1', - parentThreadId: null, - parentCallId: null, - title: null, - taskId: null, - createdAt: now, - entries: [ - entry(1, 'person', 'Find my invoices'), - entry(2, 'assistant', '', { - toolCalls: [ - { - id: 'c1', - name: 'delegate', - arguments: '{"title":"Find the invoices","brief":"Find them"}', - }, - ], - inputTokens: 1200, - outputTokens: 40, - hasSystem: true, - jobId: 'job-1', - }), - entry(3, 'tool_result', 'Started the sub-agent “Find the invoices”.', { - callId: 'c1', - jobId: 'job-2', - }), - entry(9, 'person', 'Thanks', {pending: true}), - ], - jobs: [ - job('job-1', 'thread-1'), - job('job-2', 'thread-1', {kind: 'tool', entryId: 'e2', callId: 'c1'}), - ], - events: [ - { - id: 'ev1', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'claimed', - workerId: 'host-1', - error: null, - at: '2026-10-21T09:59:58.000Z', - }, - { - id: 'ev2', - jobId: 'job-1', - threadId: 'thread-1', - kind: 'done', - workerId: 'host-1', - error: null, - at: '2026-10-21T10:00:00.500Z', - }, - ], - state, -}; -const subAgent: InspectedThread = { - id: 'thread-2', - parentThreadId: 'thread-1', - parentCallId: 'c1', - title: 'Find the invoices', - taskId: 'task-2', - createdAt: now, - entries: [ - entry(4, 'person', 'Find them'), - entry(5, 'assistant', '', { - toolCalls: [ - {id: 'c2', name: 'read', arguments: '{"connection":"gmail-1","tool":"list_emails"}'}, - ], - inputTokens: 800, - outputTokens: 30, - hasSystem: true, - jobId: 'job-3', - }), - ], - jobs: [ - job('job-3', 'thread-2'), - job('job-4', 'thread-2', { - kind: 'tool', - status: 'running', - entryId: 'e5', - callId: 'c2', - claimedBy: 'host-1', - leaseExpiresAt: '2026-10-21T10:02:00.000Z', - }), - ], - events: [ - { - id: 'ev4', - jobId: 'job-3', - threadId: 'thread-2', - kind: 'claimed', - workerId: 'host-1', - error: null, - at: '2026-10-21T09:59:50.000Z', - }, - { - id: 'ev5', - jobId: 'job-3', - threadId: 'thread-2', - kind: 'done', - workerId: 'host-1', - error: null, - at: '2026-10-21T09:59:53.000Z', - }, - { - id: 'ev3', - jobId: 'job-4', - threadId: 'thread-2', - kind: 'claimed', - workerId: 'host-1', - error: null, - at: '2026-10-21T09:59:59.000Z', - }, - ], - state: { - ...state, - pendingCalls: [{id: 'c2', name: 'read', arguments: '{}'}], - lastInputTokens: 800, - }, -}; - -const inspection: Inspection = {persona, now, threads: [conversation, subAgent]}; - -const request: ModelRequest = { - kind: 'turn', - system: 'You are signed in.', - messages: [ - {role: 'system', content: 'You are signed in.'}, - {role: 'user', content: 'Find my invoices'}, - ], - tools: [{name: 'delegate', description: 'Delegates', parameters: {type: 'object'}}], -}; - -const realFetch = globalThis.fetch; -const fetchMock = mock<(input: string, init?: RequestInit) => Promise>(); - -beforeEach(() => { - fetchMock.mockReset(); - fetchMock.mockImplementation((input) => { - if (input.endsWith('/request')) { - return Promise.resolve(Response.json(request)); - } - if (input.endsWith('/inspect')) { - return Promise.resolve(Response.json(inspection)); - } - return Promise.resolve(Response.json({error: 'unexpected'}, {status: 500})); - }); - globalThis.fetch = fetchMock as unknown as typeof fetch; -}); - -afterEach(() => { - cleanup(); - globalThis.fetch = realFetch; -}); - -function renderInspectPage() { - const router = createMemoryRouter( - [ - { - id: 'human', - loader: () => testHuman(), - element: , - children: [ - { - path: '/:org/inspect/:id', - loader: () => ({inspection, personas: [persona]}), - element: , - }, - ], - }, - ], - {initialEntries: ['/acme/inspect/persona-1']}, - ); - return render( - - - - - , - ); -} - -describe('InspectPage', () => { - test('lists the conversation and its sub-agents with where each stands, and the jobs open on them', async () => { - renderInspectPage(); - const threads = await screen.findByRole('region', {name: 'Threads'}); - const rows = within(threads).getAllByRole('button'); - expect(rows[0]?.textContent).toContain('Conversation'); - expect(rows[0]?.textContent).toContain('waiting'); - expect(rows[1]?.textContent).toContain('Find the invoices'); - expect(rows[1]?.textContent).toContain('working'); - expect(rows[1]?.textContent).toContain('1 open calls'); - expect( - within(threads).getByRole('link', {name: 'Task in the activity log'}).getAttribute('href'), - ).toBe('/acme/activity/tasks/task-2'); - - const jobs = screen.getByRole('region', {name: 'Jobs'}); - expect(jobs.textContent).toContain('running'); - expect(jobs.textContent).toContain('read'); - expect(jobs.textContent).toContain('host-1'); - expect(jobs.textContent).not.toContain('inference'); - fireEvent.click(within(jobs).getByRole('button', {name: 'With history'})); - expect(jobs.textContent).toContain('inference'); - expect(jobs.textContent).toContain('ran 2.5s'); - }); - - test('shows the conversation’s whole log, the calls with their arguments and what is pending', async () => { - renderInspectPage(); - const log = await screen.findByRole('list', {name: 'Log'}); - // The log's own items, not the calls listed within one. - const items = within(log) - .getAllByRole('listitem') - .filter((item) => item.parentElement === log); - expect(items.map((item) => item.textContent)).toEqual([ - expect.stringContaining('#1person'), - expect.stringContaining('delegate'), - expect.stringContaining('answers c1'), - expect.stringContaining('pending'), - ]); - expect(items[1]?.textContent).toContain('"title": "Find the invoices"'); - expect(items[1]?.textContent).toContain('1,200 in / 40 out'); - expect(screen.getByText(/nothing new for the model/)).toBeDefined(); - expect(screen.getByText('1 pending')).toBeDefined(); - }); - - test('opens a sub-agent’s log, and the request behind an answer in it', async () => { - renderInspectPage(); - const threads = await screen.findByRole('region', {name: 'Threads'}); - fireEvent.click(within(threads).getByRole('button', {name: /Find the invoices/})); - expect(screen.getByText(/waiting on read/)).toBeDefined(); - const log = screen.getByRole('list', {name: 'Log'}); - expect(log.textContent).toContain('list_emails'); - - fireEvent.click(within(log).getByRole('button', {name: 'Request'})); - await waitFor(() => { - expect(screen.getByText(/A turn: 2 messages, 1 tools/)).toBeDefined(); - }); - expect(fetchMock.mock.calls[0]?.[0]).toBe('/api/orgs/acme/chat/inspect/entries/e5/request'); - expect(screen.getByText('Find my invoices', {selector: 'pre'})).toBeDefined(); - expect(screen.getByText('delegate', {selector: 'summary'})).toBeDefined(); - expect(screen.getByText('claimed')).toBeDefined(); - expect(screen.getByText('after 3.0s')).toBeDefined(); - }); -}); diff --git a/packages/web/src/inspect/inspect-page.tsx b/packages/web/src/inspect/inspect-page.tsx deleted file mode 100644 index 399a703f..00000000 --- a/packages/web/src/inspect/inspect-page.tsx +++ /dev/null @@ -1,218 +0,0 @@ -import {ArrowUpIcon} from 'lucide-react'; -import {type FormEvent, type KeyboardEvent, useCallback, useEffect, useRef, useState} from 'react'; -import {Link, useLoaderData, useNavigate} from 'react-router'; -import {Button} from '@proxy/ui/components/button'; -import { - InputGroup, - InputGroupAddon, - InputGroupButton, - InputGroupTextarea, -} from '@proxy/ui/components/input-group'; -import {postMessage} from '@/client/harness-client'; -import type {Inspection} from '@proxy/contracts/inspect'; -import {getInspection} from '@/inspect/client'; -import {Head} from '@/components/head'; -import {ThreadLog} from '@/inspect/thread-log'; -import {RequestPane} from '@/inspect/request-pane'; -import {JobQueue, StatusBadge, ThreadTree} from '@/inspect/thread-tree'; -import {useOrgSlug} from '@/hooks/use-org'; -import {formatTime} from '@/lib/format'; -import {threadLabel, threadStatus} from '@/inspect/derive'; -import {describeFetchError} from '@/lib/loader-utils'; -import type {inspectLoader} from '@/inspect/loaders'; - -/** - * The inspector: the person's conversation with a persona as the harness has it, on a page of - * its own, for whoever works on the built-in agent. Nothing of the Chat page: the conversation's - * threads and the queue on the left, the thread picked in the middle with its whole log and an - * input to say something in the conversation, and on the right what the model was sent for the - * answer picked. Read again as the agent works, as the Chat page is. - */ - -const BUSY_POLL_MS = 1000; -const IDLE_POLL_MS = 4000; - -type Loaded = Awaited>; - -export function InspectPage() { - const loaded = useLoaderData(); - return ; -} - -function hasOpenJobs(inspection: Inspection): boolean { - return inspection.threads.some((thread) => - thread.jobs.some((job) => job.status === 'queued' || job.status === 'running'), - ); -} - -function Inspector({loaded}: {loaded: Loaded}) { - const org = useOrgSlug(); - const navigate = useNavigate(); - const [inspection, setInspection] = useState(loaded.inspection); - const [threadId, setThreadId] = useState(loaded.inspection.threads[0]?.id ?? ''); - const [entryId, setEntryId] = useState(null); - const [showHistory, setShowHistory] = useState(false); - const [text, setText] = useState(''); - const [sending, setSending] = useState(false); - const [error, setError] = useState(null); - const form = useRef(null); - const personaId = loaded.inspection.persona.id; - const busy = hasOpenJobs(inspection); - - const latestRead = useRef(0); - const read = useCallback(async () => { - latestRead.current += 1; - const own = latestRead.current; - const result = await getInspection(org, personaId); - if (own !== latestRead.current || result.isErr()) { - return; - } - setInspection(result.value); - }, [org, personaId]); - - useEffect(() => { - const timer = setInterval(() => void read(), busy ? BUSY_POLL_MS : IDLE_POLL_MS); - return () => { - clearInterval(timer); - }; - }, [read, busy]); - - const thread = - inspection.threads.find((candidate) => candidate.id === threadId) ?? inspection.threads[0]; - const entry = thread?.entries.find((candidate) => candidate.id === entryId) ?? null; - - const send = async (event: FormEvent) => { - event.preventDefault(); - const message = text.trim(); - if (message === '' || sending) { - return; - } - setSending(true); - setError(null); - const posted = await postMessage(org, personaId, message); - setSending(false); - if (posted.isErr()) { - setError(describeFetchError(posted.error)); - return; - } - setText(''); - await read(); - }; - - const onKeyDown = (event: KeyboardEvent) => { - if (event.key === 'Enter' && !event.shiftKey) { - event.preventDefault(); - form.current?.requestSubmit(); - } - }; - - const persona = inspection.persona; - return ( -
    -
    - - Inspector - - - Open in Chat - - - - {busy ? 'live, every second' : 'idle, every 4 seconds'} - - {formatTime(inspection.now)} - -
    -
    - -
    - {thread && ( - <> -
    - {threadLabel(thread)} - - {thread.id} - {thread.parentCallId !== null && ( - - from {thread.parentCallId} - - )} -
    - - {thread.parentThreadId === null && ( -
    void send(event)} - className="flex shrink-0 flex-col gap-1 border-t p-3" - > - {error &&

    {error}

    } - - setText(event.target.value)} - onKeyDown={onKeyDown} - placeholder={`Message ${persona.name}`} - aria-label="Message" - className="min-h-9 p-2 text-sm" - rows={1} - /> - - - - - - -
    - )} - - )} -
    - -
    -
    - ); -} diff --git a/packages/web/src/inspect/loaders.ts b/packages/web/src/inspect/loaders.ts deleted file mode 100644 index 781ad68e..00000000 --- a/packages/web/src/inspect/loaders.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {redirect} from 'react-router'; -import {listPersonas} from '@/client/harness-client'; -import {getInspection} from '@/inspect/client'; -import {unwrapLoaderResult} from '@/lib/loader-utils'; -import {type OrgParams, orgOf} from '@/loaders'; - -// The inspector of the conversation with a persona: the first persona's where the address names none. -export async function inspectIndexLoader(args: OrgParams) { - const org = orgOf(args); - const {personas} = unwrapLoaderResult(await listPersonas(org)); - const first = personas[0]; - if (!first) { - throw redirect(`/${org}`); - } - throw redirect(`/${org}/inspect/${first.id}`); -} - -// The conversation with a persona as the harness has it, and the personas to switch between. -export async function inspectLoader(args: OrgParams & {params: {id?: string}}) { - const org = orgOf(args); - const [inspection, personas] = await Promise.all([ - getInspection(org, args.params.id ?? ''), - listPersonas(org), - ]); - return { - inspection: unwrapLoaderResult(inspection), - personas: unwrapLoaderResult(personas).personas, - }; -} diff --git a/packages/web/src/inspect/request-pane.tsx b/packages/web/src/inspect/request-pane.tsx deleted file mode 100644 index 6132e811..00000000 --- a/packages/web/src/inspect/request-pane.tsx +++ /dev/null @@ -1,207 +0,0 @@ -import {useEffect, useState} from 'react'; -import {Badge} from '@proxy/ui/components/badge'; -import {cn} from '@proxy/ui/lib/utils'; -import type { - InspectedEntry, - InspectedThread, - ModelMessage, - ModelRequest, -} from '@proxy/contracts/inspect'; -import {getRequest} from '@/inspect/client'; -import {useOrgSlug} from '@/hooks/use-org'; -import {formatTime} from '@/lib/format'; -import {formatDuration, prettyJson, timedEvents} from '@/inspect/derive'; - -// The inspector's right column: what the model was sent for the answer picked in the log, as the -// api rebuilds it, the system prompt, the messages and the tools; what came back; and what -// happened to the job that ran it. - -const ROLE_CLASS: Record = { - system: 'bg-muted text-muted-foreground', - user: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', - assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', - tool: 'bg-muted text-muted-foreground', -}; - -function Section({title, children}: {title: string; children: React.ReactNode}) { - return ( -
    -

    {title}

    - {children} -
    - ); -} - -function MessageCard({message}: {message: ModelMessage}) { - const calls = message.role === 'assistant' ? (message.tool_calls ?? []) : []; - const content = message.role === 'tool' ? message.content : (message.content ?? ''); - return ( -
  • -
    - - {message.role} - - {message.role === 'tool' && {message.tool_call_id}} - {content.length.toLocaleString('en-US')} chars -
    - {content !== '' && ( -
    - - {content.slice(0, 80).replace(/\s+/g, ' ')}… - - 
    {content}
    -
    - )} - {calls.map((call) => ( -
    -
    - {call.function.name} - {call.id} -
    - 
    -						{prettyJson(call.function.arguments)}
-
    -
    - ))} -
    • - ); -} - -// What happened to the job that wrote the entry: each claim, requeue, finish and failure. -function JobHistory({entry, thread}: {entry: InspectedEntry; thread: InspectedThread}) { - const job = thread.jobs.find((candidate) => candidate.id === entry.jobId); - const events = thread.events.filter((event) => event.jobId === entry.jobId); - if (!job) { - return

    No job is kept for this entry.

    ; - } - return ( -
      -
    • - {job.id} · {job.kind} · {job.status} · {job.attempts}{' '} - {job.attempts === 1 ? 'attempt' : 'attempts'} -
      • - {timedEvents(events).map(({event, sinceClaimMs}) => ( -
    • - {formatTime(event.at)} - {event.kind} - {event.workerId !== null && ( - {event.workerId} - )} - {sinceClaimMs !== null && ( - after {formatDuration(sinceClaimMs)} - )} - {event.error !== null && {event.error}} -
      • - ))} -
    - ); -} - -export function RequestPane({ - entry, - thread, -}: { - entry: InspectedEntry | null; - thread: InspectedThread; -}) { - const org = useOrgSlug(); - const [request, setRequest] = useState<{ - entryId: string; - result: ModelRequest | null; - error: string | null; - } | null>(null); - - useEffect(() => { - if (!entry) { - return; - } - let stale = false; - void getRequest(org, entry.id).then((result) => { - if (stale) { - return; - } - setRequest({ - entryId: entry.id, - result: result.isOk() ? result.value : null, - error: result.isErr() ? 'The request could not be read.' : null, - }); - }); - return () => { - stale = true; - }; - }, [org, entry]); - - if (!entry) { - return ( -

    - Pick an answer or a summary in the log to see what the model was sent for it. -

    - ); - } - const loaded = request?.entryId === entry.id ? request : null; - return ( -
    -
    - {loaded === null &&

    Reading…

    } - {loaded?.error !== null && loaded?.error !== undefined && ( -

    {loaded.error}

    - )} - {loaded?.result && loaded.result.system === null && ( -

    - This answer was written without the model, so there was no request; the messages are the - log as it was. -

    - )} - {loaded?.result && ( -

    - {loaded.result.kind === 'summary' ? 'A summarization' : 'A turn'}:{' '} - {loaded.result.messages.length} messages, {loaded.result.tools.length} tools -

    - )} -
    - {loaded?.result && ( - <> -
    -
      - {loaded.result.messages.map((message, index) => ( - - ))} -
    -
    - {loaded.result.tools.length > 0 && ( -
    -
      - {loaded.result.tools.map((tool) => ( -
    • -
      - {tool.name} -

      {tool.description}

      - 
      -												{JSON.stringify(tool.parameters, null, 2)}
-
      -
      -
      • - ))} -
    -
    - )} - - )} -
    -

    - {(entry.inputTokens ?? 0).toLocaleString('en-US')} tokens in,{' '} - {(entry.outputTokens ?? 0).toLocaleString('en-US')} out - {entry.toolCalls && - entry.toolCalls.length > 0 && - `, ${entry.toolCalls.length} ${entry.toolCalls.length === 1 ? 'call' : 'calls'}`} -

    - {entry.text !== '' && ( - 
    {entry.text}
    - )} -
    -
    - -
    -
    - ); -} diff --git a/packages/web/src/inspect/thread-log.tsx b/packages/web/src/inspect/thread-log.tsx deleted file mode 100644 index 7658ef4c..00000000 --- a/packages/web/src/inspect/thread-log.tsx +++ /dev/null @@ -1,196 +0,0 @@ -import {Badge} from '@proxy/ui/components/badge'; -import {Button} from '@proxy/ui/components/button'; -import {cn} from '@proxy/ui/lib/utils'; -import type {InspectedEntry, InspectedThread, ThreadState} from '@proxy/contracts/inspect'; -import {formatTime} from '@/lib/format'; -import {coveredSeq, prettyJson} from '@/inspect/derive'; - -// The inspector's middle: a thread's whole log as the harness has it, every entry with what the -// model doesn't see (its seq, its cost, its job), and above it where the thread stands for the -// harness: whether the model is due, what holds it, how close the log is to being summarized. - -// Text longer than this is folded, with its length shown. -const FOLD_AT = 600; - -const KIND_CLASS: Record = { - person: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', - assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', - tool_result: 'bg-muted text-muted-foreground', - event: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', - summary: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', -}; - -function Folded({text, open = false}: {text: string; open?: boolean}) { - if (text.length <= FOLD_AT) { - return 
    {text}
    ; - } - return ( -
    - - {text.length.toLocaleString('en-US')} characters - - 
    {text}
    -
    - ); -} - -function Calls({entry}: {entry: InspectedEntry}) { - const calls = entry.toolCalls ?? []; - if (calls.length === 0) { - return null; - } - return ( -
      - {calls.map((call) => ( -
    • -
      - {call.name} - {call.id} -
      - 
      -						{prettyJson(call.arguments)}
-
      -
      • - ))} -
    - ); -} - -// What an entry says beside its kind: which call it answers, how far a summary reaches. -function lead(entry: InspectedEntry): string | null { - if (entry.kind === 'tool_result') { - return `answers ${entry.callId ?? '?'}`; - } - if (entry.kind === 'event') { - return `report for ${entry.callId ?? '?'}`; - } - if (entry.kind === 'summary') { - return `covers up to #${entry.coversSeq ?? '?'}`; - } - return null; -} - -function EntryCard({ - entry, - covered, - selected, - onSelect, -}: { - entry: InspectedEntry; - covered: boolean; - selected: boolean; - onSelect: (entryId: string) => void; -}) { - const askable = entry.kind === 'assistant' || entry.kind === 'summary'; - const tokens = entry.inputTokens !== null || entry.outputTokens !== null; - return ( -
  • -
    - #{entry.seq} - - {entry.kind} - - {lead(entry) && {lead(entry)}} - {formatTime(entry.at)} - {tokens && ( - - {(entry.inputTokens ?? 0).toLocaleString('en-US')} in /{' '} - {(entry.outputTokens ?? 0).toLocaleString('en-US')} out - - )} - {entry.pending && pending} - {entry.isError && error} - {covered && summarized} - {entry.jobId !== null && job {entry.jobId.slice(-6)}} - {askable && ( - - )} -
    - {entry.text !== '' && } - -
    • - ); -} - -// Where the thread stands for the harness. -function StateStrip({state, pending}: {state: ThreadState; pending: number}) { - const share = Math.min(1, state.lastInputTokens / state.compactAtTokens); - const holds = - state.pendingCalls.length > 0 - ? `waiting on ${state.pendingCalls.map((call) => call.name).join(', ')}` - : state.ready - ? 'model due' - : 'nothing new for the model'; - return ( -
    - {holds} - {pending > 0 && {pending} pending} - - {state.turnsAlone}/{state.maxTurnsAlone} steps alone - - - - {state.lastInputTokens.toLocaleString('en-US')}/ - {state.compactAtTokens.toLocaleString('en-US')} tokens - - - - - - {state.compactionCoversSeq !== null && ( - next turn summarizes up to #{state.compactionCoversSeq} - )} -
    - ); -} - -export function ThreadLog({ - thread, - selectedEntryId, - onSelectEntry, -}: { - thread: InspectedThread; - selectedEntryId: string | null; - onSelectEntry: (entryId: string) => void; -}) { - const covered = coveredSeq(thread.entries); - const pending = thread.entries.filter((entry) => entry.pending).length; - return ( - <> - -
    -
      - {thread.entries.length === 0 && ( -
    • - Nothing in the log yet. -
      • - )} - {thread.entries.map((entry) => ( - - ))} -
    -
    - - ); -} diff --git a/packages/web/src/inspect/thread-tree.tsx b/packages/web/src/inspect/thread-tree.tsx deleted file mode 100644 index 370687a7..00000000 --- a/packages/web/src/inspect/thread-tree.tsx +++ /dev/null @@ -1,250 +0,0 @@ -import {Link} from 'react-router'; -import {Badge} from '@proxy/ui/components/badge'; -import {cn} from '@proxy/ui/lib/utils'; -import type {InspectedJob, InspectedThread} from '@proxy/contracts/inspect'; -import {useOrgSlug} from '@/hooks/use-org'; -import {formatTime} from '@/lib/format'; -import { - formatDue, - formatDuration, - jobTitle, - lastAttempt, - threadLabel, - threadStatus, - type ThreadStatus, - tokensOf, -} from '@/inspect/derive'; - -// The inspector's left column: the conversation's threads, the conversation first and the -// sub-agents started from it under it, each with where it stands; and under them the queue, -// every job of every thread still to run or running, with its history behind a toggle. - -const STATUS_LABEL: Record = { - working: 'working', - queued: 'queued', - asleep: 'asleep', - waiting: 'waiting', - stopped: 'stopped', - failed: 'failed', -}; - -const STATUS_CLASS: Record = { - working: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', - queued: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', - asleep: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', - waiting: 'bg-muted text-muted-foreground', - stopped: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', - failed: 'bg-destructive/10 text-destructive', -}; - -export function StatusBadge({status}: {status: ThreadStatus}) { - return ( - - {STATUS_LABEL[status.kind]} - {status.until !== null && ` until ${formatTime(status.until)}`} - - ); -} - -function ThreadRow({ - thread, - now, - current, - onSelect, -}: { - thread: InspectedThread; - now: string; - current: boolean; - onSelect: (threadId: string) => void; -}) { - const org = useOrgSlug(); - const tokens = tokensOf(thread.entries); - const sub = thread.parentThreadId !== null; - return ( -
  • - - {thread.taskId !== null && ( - - Task in the activity log - - )} -
    • - ); -} - -export function ThreadTree({ - threads, - now, - current, - onSelect, -}: { - threads: InspectedThread[]; - now: string; - current: string; - onSelect: (threadId: string) => void; -}) { - return ( -
    -

    - Threads -

    -
      - {threads.map((thread) => ( - - ))} -
    -
    - ); -} - -const JOB_CLASS: Record = { - running: STATUS_CLASS.working, - queued: STATUS_CLASS.queued, - done: STATUS_CLASS.stopped, - failed: STATUS_CLASS.failed, -}; - -// Running first, then queued by when they are due, then what is over, newest first. -function order(a: InspectedJob, b: InspectedJob): number { - const rank = (job: InspectedJob) => - job.status === 'running' ? 0 : job.status === 'queued' ? 1 : 2; - if (rank(a) !== rank(b)) { - return rank(a) - rank(b); - } - if (a.status === 'queued') { - return a.runAt.localeCompare(b.runAt); - } - return b.createdAt.localeCompare(a.createdAt); -} - -// What a job is up to, in a line: due when, held by whom until when, or how long it ran. -function jobDetail(job: InspectedJob, thread: InspectedThread, now: string): string { - if (job.status === 'queued') { - return `${formatDue(job.runAt, now)}${job.attempts > 0 ? `, attempt ${job.attempts + 1}` : ''}`; - } - const attempt = lastAttempt(job, thread.events); - if (job.status === 'running') { - const lease = - job.leaseExpiresAt === null - ? '' - : `, lease ${formatDue(job.leaseExpiresAt, now).replace('in ', '')} left`; - const since = - attempt.claimedAt === null - ? '' - : `, ${formatDuration(new Date(now).getTime() - new Date(attempt.claimedAt).getTime())} so far`; - return `${job.claimedBy ?? 'claimed'}${lease}${since}, attempt ${job.attempts}`; - } - const ran = attempt.ranMs === null ? '' : `ran ${formatDuration(attempt.ranMs)}, `; - return `${ran}${job.attempts} ${job.attempts === 1 ? 'attempt' : 'attempts'}`; -} - -function JobRow({ - job, - thread, - now, - onSelectThread, -}: { - job: InspectedJob; - thread: InspectedThread; - now: string; - onSelectThread: (threadId: string) => void; -}) { - return ( -
  • - - - {job.status} - - {jobTitle(job, thread.entries)} - - - {jobDetail(job, thread, now)} - {job.error !== null && {job.error}} -
    • - ); -} - -export function JobQueue({ - threads, - now, - showHistory, - onToggleHistory, - onSelectThread, -}: { - threads: InspectedThread[]; - now: string; - showHistory: boolean; - onToggleHistory: () => void; - onSelectThread: (threadId: string) => void; -}) { - const jobs = threads - .flatMap((thread) => thread.jobs.map((job) => ({job, thread}))) - .filter(({job}) => showHistory || job.status === 'queued' || job.status === 'running') - .sort((a, b) => order(a.job, b.job)); - return ( -
    -

    - Jobs - -

    - {jobs.length === 0 && ( -

    Nothing queued or running.

    - )} -
      - {jobs.map(({job, thread}) => ( - - ))} -
    -
    - ); -} diff --git a/packages/web/src/loaders.ts b/packages/web/src/loaders.ts index 844002d0..306dad28 100644 --- a/packages/web/src/loaders.ts +++ b/packages/web/src/loaders.ts @@ -26,11 +26,9 @@ import {redirect, type ShouldRevalidateFunctionArgs} from 'react-router'; // page is in an organization, named by its slug in the address (`params.org`), which its api // calls are about. -/** The route params of a page in an organization. */ -export type OrgParams = {params: {org?: string}}; +type OrgParams = {params: {org?: string}}; -/** The organization's slug from the address. */ -export function orgOf({params}: OrgParams): string { +function orgOf({params}: OrgParams): string { return params.org ?? ''; } diff --git a/packages/web/src/router.tsx b/packages/web/src/router.tsx index 945f04e9..252e843f 100644 --- a/packages/web/src/router.tsx +++ b/packages/web/src/router.tsx @@ -65,8 +65,6 @@ import {ConnectionsPage} from '@/pages/connections-page'; import {DesktopAuthPage} from '@/pages/desktop-auth-page'; import {DesktopReturnPage} from '@/pages/desktop-return-page'; import {HomePage} from '@/pages/home-page'; -import {inspectIndexLoader, inspectLoader} from '@/inspect/loaders'; -import {InspectPage} from '@/inspect/inspect-page'; import {InvitePage} from '@/pages/invite-page'; import {LoginPage} from '@/pages/login-page'; import {MembersPage} from '@/pages/members-page'; @@ -291,10 +289,6 @@ export const router = createBrowserRouter([ // The built-in agents, behind the `harness` experiment: the api refuses it otherwise. {path: '/:org/chat', loader: chatIndexLoader, element: }, {path: '/:org/chat/:id', loader: chatLoader, element: }, - // The inspector of the built-in agent, behind the `debug` experiment too, on a - // page of its own, reached by its address alone. - {path: '/:org/inspect', loader: inspectIndexLoader, element: }, - {path: '/:org/inspect/:id', loader: inspectLoader, element: }, {path: '/:org/agents', loader: agentsLoader, element: }, {path: '/:org/agents/new', loader: agentsLoader, element: }, {path: '/:org/agents/:id', loader: agentLoader, element: },