diff --git a/packages/api/src/harness/inference.ts b/packages/api/src/harness/inference.ts index 45b463f9..0e4fcbd4 100644 --- a/packages/api/src/harness/inference.ts +++ b/packages/api/src/harness/inference.ts @@ -50,14 +50,14 @@ const SUB_FLOW = // How a long log is summarized, by the same model, for itself: a conversation's, or a // sub-agent's, whose one message from the person is its brief. -const SUMMARIZE = +export const SUMMARIZE = 'You are summarizing a conversation between a person and their agent, written out below with the calls the agent made and what they answered, so that the agent can go on from the summary in place of the conversation so far. Write what the agent needs to go on as if it remembered: what the person asked for and what they said they want in general; what was done and found, with the names, ids, dates and numbers that matter; what the person was told; what is still open, promised or planned, such as what to do after a sleep or when a sub-agent reports. Leave out what no longer matters. Plain prose, from the agent’s side, no title.'; -const SUMMARIZE_NOW = 'Write the summary now.'; +export const SUMMARIZE_NOW = 'Write the summary now.'; // How many answers the model may give in a row without the person saying anything, calls, // reports and wake-ups included, before it stops and waits for them; a report that comes after // the stop is acted on, and the count starts over. -const MAX_TURNS_ALONE = 30; +export const MAX_TURNS_ALONE = 30; // How often a model that can't be reached is tried, and how long to wait between tries. const MAX_MODEL_ATTEMPTS = 3; @@ -80,9 +80,8 @@ function isStop(entry: EntryRow): boolean { return entry.kind === 'assistant' && (entry.text === TOO_LONG || entry.text === TOO_LONG_SUB); } -// How many answers the model has given since the person last spoke, or since it last stopped for -// them. -function turnsAlone(entries: EntryRow[]): number { +/** How many answers the model has given since the person last spoke, or since it last stopped for them. */ +export function turnsAlone(entries: EntryRow[]): number { const since = entries.findLastIndex((entry) => entry.kind === 'person' || isStop(entry)); return entries.slice(since + 1).filter((entry) => entry.kind === 'assistant').length; } diff --git a/packages/api/src/harness/inspect-route.test.ts b/packages/api/src/harness/inspect-route.test.ts new file mode 100644 index 00000000..a2aa8eba --- /dev/null +++ b/packages/api/src/harness/inspect-route.test.ts @@ -0,0 +1,252 @@ +import {beforeEach, describe, expect, mock, test} from 'bun:test'; +import {Err, Ok} from 'ts-results-es'; + +import {ApiErr} from '@proxy/utils'; + +mock.module('../utils/env', () => ({env: {NODE_ENV: 'test', APP_URL: 'http://localhost:5173'}})); + +const agentInstructions = mock(async (_agent: unknown, flow: string) => Ok(`You are signed in.\n\n${flow}`)); +mock.module('../mcp/tools', () => ({agentInstructions, MCP_TOOLS: []})); + +const delegate = {name: 'delegate', description: 'Delegates', parameters: {}}; +const read = {name: 'read', description: 'Reads', parameters: {}}; +mock.module('./tool-surface', () => ({toolsFor: (candidate: {parentThreadId: string | null}) => (candidate.parentThreadId === null ? [delegate] : [read])})); + +const getPersona = mock(); +const getMemberThread = mock(); +const listSubAgentThreads = mock(); +const listThreadJobs = mock(); +const listJobEvents = mock(); +const listEntriesWithPending = mock(); +const listEntries = mock(); +const getEntry = mock(); +const getJobThread = mock(); +// What the route reads, and what the inference module it takes its constants from imports. +mock.module('@proxy/db/harness', () => ({ + getPersona, + getMemberThread, + listSubAgentThreads, + listThreadJobs, + listJobEvents, + listEntriesWithPending, + listEntries, + getEntry, + getJobThread, + appendSummary: mock(), + completeInference: mock(), + finishJob: mock(), + getThreadAgent: mock(), + requeueJob: mock(), +})); + +const viewer = {orgId: 'org-1', userId: 'user-1', slug: 'acme', name: 'Acme', role: 'member' as 'member' | 'owner', onboarded: true, experiments: ['harness', 'debug']}; +const requireViewer = mock(async () => Ok(viewer)); +mock.module('../utils/user-org', () => ({requireViewer})); + +const now = new Date('2026-10-21T10:00:00Z'); +const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; +const persona = {id: 'persona-1', name: 'Finance', look, color: 'blue', createdById: 'user-1'}; +const thread = { + id: 'thread-1', + orgId: 'org-1', + agentId: 'login-1', + personaId: 'persona-1', + userId: 'user-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, +}; +const sub = {...thread, id: 'thread-2', parentThreadId: 'thread-1', parentCallId: 'c1', title: 'Find the invoices', taskId: 'task-2'}; + +type Kind = 'person' | 'assistant' | 'tool_result' | 'event' | 'summary'; + +function entry( + seq: number, + threadId: string, + kind: Kind, + text: string, + extra: Partial<{toolCalls: Array<{id: string; name: string; arguments: string}>; callId: string; inputTokens: number; coversSeq: number; system: string; jobId: string; pending: boolean}> = {}, +) { + return { + id: `e${seq}`, + seq, + threadId, + kind, + pending: extra.pending ?? false, + text, + toolCalls: extra.toolCalls ?? null, + callId: extra.callId ?? null, + isError: false, + inputTokens: extra.inputTokens ?? null, + outputTokens: null, + coversSeq: extra.coversSeq ?? null, + system: extra.system ?? null, + jobId: extra.jobId ?? null, + createdAt: now, + }; +} + +// The person asked, the agent delegated, and the sub-agent has read once and is waiting on it. +const conversation = [ + entry(1, 'thread-1', 'person', 'Find my invoices'), + entry(2, 'thread-1', 'assistant', '', { + toolCalls: [{id: 'c1', name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}], + inputTokens: 1200, + system: 'You are signed in.', + jobId: 'job-1', + }), + entry(3, 'thread-1', 'tool_result', 'Started the sub-agent “Find the invoices”.', {callId: 'c1', jobId: 'job-2'}), + entry(4, 'thread-1', 'assistant', 'Looking.', {inputTokens: 1300, system: 'You are signed in.', jobId: 'job-3'}), + entry(7, 'thread-1', 'person', 'Thanks', {pending: true}), +]; +const subLog = [ + entry(5, 'thread-2', 'person', 'Find them'), + entry(6, 'thread-2', 'assistant', '', {toolCalls: [{id: 'c2', name: 'read', arguments: '{}'}], inputTokens: 800, system: 'You are a sub-agent.', jobId: 'job-4'}), +]; +const jobs = [ + { + id: 'job-1', + threadId: 'thread-1', + kind: 'inference' as const, + status: 'done' as const, + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 1, + error: null, + createdAt: now, + }, + { + id: 'job-5', + threadId: 'thread-2', + kind: 'tool' as const, + status: 'running' as const, + entryId: 'e6', + callId: 'c2', + runAt: now, + claimedBy: 'host-1', + leaseExpiresAt: now, + attempts: 1, + error: null, + createdAt: now, + }, +]; +const events = [ + {id: 'ev1', jobId: 'job-1', threadId: 'thread-1', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, + {id: 'ev2', jobId: 'job-5', threadId: 'thread-2', kind: 'claimed' as const, workerId: 'host-1', error: null, at: now}, +]; + +function request(params: Record = {}) { + return {user: {userId: 'user-1', email: 'a@example.com'}, params, body: undefined} as never; +} + +beforeEach(() => { + mock.clearAllMocks(); + requireViewer.mockResolvedValue(Ok(viewer)); + getPersona.mockResolvedValue(Ok(persona)); + getMemberThread.mockResolvedValue(Ok(thread)); + listSubAgentThreads.mockResolvedValue(Ok([sub])); + listThreadJobs.mockResolvedValue(Ok(jobs)); + listJobEvents.mockResolvedValue(Ok(events)); + listEntriesWithPending.mockImplementation(async (threadId: string) => Ok(threadId === 'thread-1' ? conversation : subLog)); + listEntries.mockImplementation(async (threadId: string) => Ok((threadId === 'thread-1' ? conversation : subLog).filter((candidate) => !candidate.pending))); + getEntry.mockImplementation(async (id: string) => { + const found = [...conversation, ...subLog].find((candidate) => candidate.id === id); + return found ? Ok(found) : Err(ApiErr.notFound('entry', id)); + }); + getJobThread.mockImplementation(async (id: string) => Ok(id === 'thread-1' ? thread : id === 'thread-2' ? sub : null)); +}); + +describe('the inspection of a conversation', () => { + test('is refused without the inspector experiment, as the Chat page is without its own', async () => { + const {handleInspectThreadRoute, handleInspectRequestRoute} = await import('./inspect-route'); + requireViewer.mockResolvedValue(Ok({...viewer, experiments: ['harness']})); + expect((await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('forbidden'); + expect((await handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('forbidden'); + }); + + test('is the conversation first, then its sub-agents, each with its whole log, its jobs and their events', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + const result = (await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap(); + expect(result.persona.id).toBe('persona-1'); + expect(result.threads.map((candidate) => candidate.id)).toEqual(['thread-1', 'thread-2']); + expect(listThreadJobs).toHaveBeenCalledWith(['thread-1', 'thread-2']); + expect(listJobEvents).toHaveBeenCalledWith(['thread-1', 'thread-2']); + const [main, agent] = result.threads; + expect(main?.entries.map((candidate) => candidate.kind)).toEqual(['person', 'assistant', 'tool_result', 'assistant', 'person']); + expect(main?.entries.at(-1)?.pending).toBe(true); + expect(main?.entries[1]).toMatchObject({hasSystem: true, jobId: 'job-1', inputTokens: 1200, toolCalls: [{id: 'c1', name: 'delegate', arguments: expect.any(String)}]}); + expect(main?.jobs.map((candidate) => candidate.id)).toEqual(['job-1']); + expect(main?.events.map((candidate) => candidate.id)).toEqual(['ev1']); + expect(agent?.title).toBe('Find the invoices'); + expect(agent?.jobs.map((candidate) => candidate.id)).toEqual(['job-5']); + expect(agent?.jobs[0]).toMatchObject({status: 'running', claimedBy: 'host-1', leaseExpiresAt: now.toISOString()}); + }); + + test('says where each thread stands: the conversation waits for the pending turn to land, the sub-agent for its call', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + const [main, agent] = (await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrap().threads; + // The pending turn is not in the log yet: the log ends with an answer, so the model is not due. + expect(main?.state).toMatchObject({ready: false, pendingCalls: [], turnsAlone: 2, maxTurnsAlone: 30, lastInputTokens: 1300, compactAtTokens: 80_000, compactionCoversSeq: null}); + expect(agent?.state).toMatchObject({ready: false, pendingCalls: [{id: 'c2', name: 'read'}], turnsAlone: 1, lastInputTokens: 800}); + }); + + test('is not found before the viewer has a conversation with the persona', async () => { + const {handleInspectThreadRoute} = await import('./inspect-route'); + getMemberThread.mockResolvedValue(Ok(null)); + expect((await handleInspectThreadRoute(request({personaId: 'persona-1'}))).unwrapErr().kind).toBe('not_found'); + }); +}); + +describe('the request behind an entry', () => { + test('is the system prompt the answer kept, the log before it as the model read it, and the thread’s tools', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + const result = (await handleInspectRequestRoute(request({entryId: 'e4'}))).unwrap(); + expect(result.kind).toBe('turn'); + expect(result.system).toBe('You are signed in.'); + expect(result.messages).toEqual([ + {role: 'system', content: 'You are signed in.'}, + {role: 'user', content: 'Find my invoices'}, + {role: 'assistant', content: null, tool_calls: [{id: 'c1', type: 'function', function: {name: 'delegate', arguments: '{"title":"Find the invoices","brief":"Find them"}'}}]}, + {role: 'tool', tool_call_id: 'c1', content: 'Started the sub-agent “Find the invoices”.'}, + ]); + expect(result.tools).toEqual([delegate]); + }); + + test('has a sub-agent’s tools on a sub-agent’s thread', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + const result = (await handleInspectRequestRoute(request({entryId: 'e6'}))).unwrap(); + expect(result.messages).toEqual([ + {role: 'system', content: 'You are a sub-agent.'}, + {role: 'user', content: 'Find them'}, + ]); + expect(result.tools).toEqual([read]); + }); + + test('is the transcript of what a summary covers, for the model to summarize', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + getEntry.mockResolvedValue(Ok(entry(5, 'thread-1', 'summary', 'The person asked for invoices.', {coversSeq: 3}))); + const result = (await handleInspectRequestRoute(request({entryId: 'e5'}))).unwrap(); + expect(result.kind).toBe('summary'); + expect(result.tools).toEqual([]); + expect(result.messages[0]?.role).toBe('system'); + const asked = result.messages[1]; + expect(asked?.role).toBe('user'); + expect(asked?.content).toContain('Person:\nFind my invoices'); + expect(asked?.content).toContain('Agent called delegate with'); + expect(asked?.content).not.toContain('Looking.'); + expect(asked?.content).toMatch(/Write the summary now\.$/); + }); + + test('is not found for an entry of someone else’s thread, or one that is neither an answer nor a summary', async () => { + const {handleInspectRequestRoute} = await import('./inspect-route'); + expect((await handleInspectRequestRoute(request({entryId: 'e1'}))).unwrapErr().kind).toBe('not_found'); + getJobThread.mockResolvedValue(Ok({...thread, userId: 'user-2'})); + expect((await handleInspectRequestRoute(request({entryId: 'e2'}))).unwrapErr().kind).toBe('not_found'); + }); +}); diff --git a/packages/api/src/harness/inspect-route.ts b/packages/api/src/harness/inspect-route.ts new file mode 100644 index 00000000..618c4738 --- /dev/null +++ b/packages/api/src/harness/inspect-route.ts @@ -0,0 +1,260 @@ +import {Err, type Result} from 'ts-results-es'; + +import {ApiErr, type ApiError, Do, hasExperiment, requirePresent} from '@proxy/utils'; +import {pendingToolCalls, readyForInference, type ToolCall} from '@proxy/utils/harness'; +import { + type EntryRow, + getEntry, + getJobThread, + getMemberThread, + getPersona, + type JobDetailRow, + type JobEventRow, + listEntries, + listEntriesWithPending, + listJobEvents, + listSubAgentThreads, + listThreadJobs, + type ThreadRow, +} from '@proxy/db/harness'; + +import type {AuthenticatedRequest} from '../server/middleware/require-auth'; +import {type OrgViewer, requireViewer} from '../utils/user-org'; +import {COMPACT_AT_TOKENS, compactionCut, toMessages, toTranscript} from './context'; +import {MAX_TURNS_ALONE, SUMMARIZE, SUMMARIZE_NOW, turnsAlone} from './inference'; +import type {ModelMessage, ModelTool} from './model'; +import {type PersonaResponse, toPersona} from './persona-response'; +import {toolsFor} from './tool-surface'; + +/** + * The inspector's routes, for whoever works on the built-in agent, behind the `debug` + * experiment beside `harness`: the viewer's conversation with a persona as the harness has it, + * every thread of it with its whole log, its jobs and what happened to each, and, for any answer + * of the model, the request it was sent, rebuilt from the log with the functions the harness runs + * (context.ts) and the system prompt the entry kept. The viewer sees their own conversations + * alone, as on the Chat page: a conversation is theirs, whoever they are in the organization. + */ + +/** An entry of a thread's log, whole: every kind, what is pending too, with what the model's answer cost and whether its system prompt was kept, which `GET …/inspect/entries/:entryId/request` gives with the rest of the request. */ +export type InspectedEntry = { + id: string; + seq: number; + kind: EntryRow['kind']; + text: string; + toolCalls: ToolCall[] | null; + callId: string | null; + isError: boolean; + pending: boolean; + inputTokens: number | null; + outputTokens: number | null; + coversSeq: number | null; + hasSystem: boolean; + jobId: string | null; + at: string; +}; + +/** A job as the queue has it, whatever its status. */ +export type InspectedJob = { + id: string; + threadId: string; + kind: JobDetailRow['kind']; + status: JobDetailRow['status']; + entryId: string | null; + callId: string | null; + runAt: string; + claimedBy: string | null; + leaseExpiresAt: string | null; + attempts: number; + error: string | null; + createdAt: string; +}; + +/** What happened to a job, one row per happening. */ +export type InspectedJobEvent = {id: string; jobId: string; threadId: string; kind: JobEventRow['kind']; workerId: string | null; error: string | null; at: string}; + +/** Where the thread stands for the harness: whether its log is ready for the model and what holds it back, how many steps the model took alone against the limit, and how close the log is to being summarized. */ +export type ThreadState = { + ready: boolean; + pendingCalls: ToolCall[]; + turnsAlone: number; + maxTurnsAlone: number; + lastInputTokens: number; + compactAtTokens: number; + // The seq of the last entry a summary made now would cover; null when the log needs none. + compactionCoversSeq: number | null; +}; + +/** A thread of the conversation: the conversation itself, or a sub-agent started from it. */ +export type InspectedThread = { + id: string; + parentThreadId: string | null; + parentCallId: string | null; + title: string | null; + taskId: string | null; + finishedAt: string | null; + createdAt: string; + entries: InspectedEntry[]; + jobs: InspectedJob[]; + events: InspectedJobEvent[]; + state: ThreadState; +}; + +/** The viewer's conversation with a persona, whole: the conversation's thread first, then every sub-agent started from it, oldest first. */ +export type InspectionResponse = {persona: PersonaResponse; now: string; threads: InspectedThread[]}; + +/** What the model was sent for an entry: the system prompt as kept (null where the answer was written without the model), the messages and the tools, rebuilt from the log as it was. */ +export type RequestResponse = {kind: 'turn' | 'summary'; system: string | null; messages: ModelMessage[]; tools: ModelTool[]}; + +function toInspectedEntry(entry: EntryRow): InspectedEntry { + return { + id: entry.id, + seq: entry.seq, + kind: entry.kind, + text: entry.text, + toolCalls: entry.toolCalls, + callId: entry.callId, + isError: entry.isError, + pending: entry.pending, + inputTokens: entry.inputTokens, + outputTokens: entry.outputTokens, + coversSeq: entry.coversSeq, + hasSystem: entry.system !== null, + jobId: entry.jobId, + at: entry.createdAt.toISOString(), + }; +} + +function toInspectedJob(job: JobDetailRow): InspectedJob { + return { + id: job.id, + threadId: job.threadId, + kind: job.kind, + status: job.status, + entryId: job.entryId, + callId: job.callId, + runAt: job.runAt.toISOString(), + claimedBy: job.claimedBy, + leaseExpiresAt: job.leaseExpiresAt?.toISOString() ?? null, + attempts: job.attempts, + error: job.error, + createdAt: job.createdAt.toISOString(), + }; +} + +function toInspectedEvent(event: JobEventRow): InspectedJobEvent { + return {id: event.id, jobId: event.jobId, threadId: event.threadId, kind: event.kind, workerId: event.workerId, error: event.error, at: event.at.toISOString()}; +} + +// Where the thread stands, from the log the model reads: what is pending is not in it yet. +function stateOf(entries: EntryRow[]): ThreadState { + const log = entries.filter((entry) => !entry.pending); + const cut = compactionCut(log); + const lastAnswer = log.findLast((entry) => entry.kind === 'assistant'); + return { + ready: readyForInference(log), + pendingCalls: pendingToolCalls(log), + turnsAlone: turnsAlone(log), + maxTurnsAlone: MAX_TURNS_ALONE, + lastInputTokens: lastAnswer?.inputTokens ?? 0, + compactAtTokens: COMPACT_AT_TOKENS, + compactionCoversSeq: cut === null ? null : (log[cut - 1]?.seq ?? null), + }; +} + +function requireInspector(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireViewer(request)); + if (!hasExperiment(viewer, 'harness') || !hasExperiment(viewer, 'debug')) { + return $(Err(ApiErr.forbidden())); + } + return viewer; + }); +} + +// A thread the viewer may inspect: one of their own conversations, or a sub-agent started from one. +function mayInspect(viewer: OrgViewer, thread: ThreadRow): boolean { + return thread.orgId === viewer.orgId && thread.userId === viewer.userId; +} + +async function inspectThread(thread: ThreadRow, jobs: JobDetailRow[], events: JobEventRow[]): Promise> { + return Do(async ($) => { + const entries = $(await listEntriesWithPending(thread.id)); + return { + id: thread.id, + parentThreadId: thread.parentThreadId, + parentCallId: thread.parentCallId, + title: thread.title, + taskId: thread.taskId, + finishedAt: thread.finishedAt?.toISOString() ?? null, + createdAt: thread.createdAt.toISOString(), + entries: entries.map(toInspectedEntry), + jobs: jobs.filter((job) => job.threadId === thread.id).map(toInspectedJob), + events: events.filter((event) => event.threadId === thread.id).map(toInspectedEvent), + state: stateOf(entries), + }; + }); +} + +/** `GET /api/orgs/:org/chat/personas/:personaId/inspect`: the viewer's conversation with the persona as the harness has it; not found before they have one. */ +export function handleInspectThreadRoute(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireInspector(request)); + const personaId = request.params.personaId ?? ''; + const persona = $(requirePresent($(await getPersona(viewer.orgId, personaId)), ApiErr.notFound('persona', personaId))); + const conversation = $(requirePresent($(await getMemberThread(persona.id, viewer.userId)), ApiErr.notFound('thread', persona.id))); + const subAgents = $(await listSubAgentThreads(conversation.id)); + const threads = [conversation, ...subAgents]; + const ids = threads.map((thread) => thread.id); + const [jobs, events] = await Promise.all([listThreadJobs(ids), listJobEvents(ids)]); + const allJobs = $(jobs); + const allEvents = $(events); + const inspected: InspectedThread[] = []; + for (const thread of threads) { + inspected.push($(await inspectThread(thread, allJobs, allEvents))); + } + return {persona: toPersona(viewer, persona), now: new Date().toISOString(), threads: inspected}; + }); +} + +// The request behind an assistant entry: the log before it, as the model read it, behind the +// system prompt the entry kept. +function turnRequest(thread: ThreadRow, entry: EntryRow, log: EntryRow[]): RequestResponse { + const before = log.filter((candidate) => candidate.seq < entry.seq); + const system: ModelMessage[] = entry.system === null ? [] : [{role: 'system', content: entry.system}]; + return {kind: 'turn', system: entry.system, messages: [...system, ...toMessages(before, {clearOld: true})], tools: toolsFor(thread)}; +} + +// The request behind a summary entry: the log it covers, written out for the model to summarize. +function summaryRequest(entry: EntryRow, log: EntryRow[]): RequestResponse { + const covered = log.filter((candidate) => candidate.seq <= (entry.coversSeq ?? -1)); + return { + kind: 'summary', + system: SUMMARIZE, + messages: [ + {role: 'system', content: SUMMARIZE}, + {role: 'user', content: `${toTranscript(covered)}\n\n${SUMMARIZE_NOW}`}, + ], + tools: [], + }; +} + +/** `GET /api/orgs/:org/chat/inspect/entries/:entryId/request`: what the model was sent for an assistant or summary entry of one of the viewer's threads, rebuilt from the log; not found for any other entry. */ +export function handleInspectRequestRoute(request: AuthenticatedRequest): Promise> { + return Do(async ($) => { + const viewer = $(await requireInspector(request)); + const entryId = request.params.entryId ?? ''; + const entry = $(await getEntry(entryId)); + const thread = $(requirePresent($(await getJobThread(entry.threadId)), ApiErr.notFound('thread', entry.threadId))); + if (!mayInspect(viewer, thread)) { + return $(Err(ApiErr.notFound('entry', entryId))); + } + const log = $(await listEntries(thread.id)); + if (entry.kind === 'assistant') { + return turnRequest(thread, entry, log); + } + if (entry.kind === 'summary') { + return summaryRequest(entry, log); + } + return $(Err(ApiErr.notFound('entry', entryId))); + }); +} diff --git a/packages/api/src/harness/persona-response.ts b/packages/api/src/harness/persona-response.ts new file mode 100644 index 00000000..c0fd7edf --- /dev/null +++ b/packages/api/src/harness/persona-response.ts @@ -0,0 +1,26 @@ +import type {LastMessage, PersonaRow} from '@proxy/db/harness'; +import {type Look, preview} from '@proxy/utils/harness'; + +import {manages} from '../utils/manages'; +import type {OrgViewer} from '../utils/user-org'; + +/** A persona as the Chat page lists it, whether the viewer may change it, and the last thing said in their conversation with it, cut to a line; null before anything was said. */ +export type PersonaResponse = {id: string; name: string; look: Look; color: string; mayChange: boolean; lastMessage: {kind: 'person' | 'assistant'; text: string; at: string} | null}; + +/** Whether the viewer may rename, reface or delete the persona: whoever made it, or the organization's owner (utils/manages.ts). */ +export function managesPersona(viewer: OrgViewer, persona: PersonaRow): boolean { + return manages(viewer, {ownerUserId: null, byId: persona.createdById}); +} + +/** A persona for a response, with whether the viewer may change it. */ +export function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): PersonaResponse { + const last = persona.lastMessage ?? null; + return { + id: persona.id, + name: persona.name, + look: persona.look, + color: persona.color, + mayChange: managesPersona(viewer, persona), + lastMessage: last && {kind: last.kind, text: preview(last.text), at: last.createdAt.toISOString()}, + }; +} diff --git a/packages/api/src/harness/route.ts b/packages/api/src/harness/route.ts index 665ed795..f4384153 100644 --- a/packages/api/src/harness/route.ts +++ b/packages/api/src/harness/route.ts @@ -2,7 +2,7 @@ import {Err, type Result} from 'ts-results-es'; import {z} from 'zod'; import {ApiErr, type ApiError, Do, hasExperiment, parseSchema, requirePresent} from '@proxy/utils'; -import {AVATAR_COLORS, type Look, lookSchema, preview} from '@proxy/utils/harness'; +import {AVATAR_COLORS, lookSchema} from '@proxy/utils/harness'; import { appendPersonMessage, createPersona, @@ -12,7 +12,6 @@ import { type EntryRow, getMemberThread, getPersona, - type LastMessage, listOpenJobs, listPersonas, listShownEntries, @@ -23,9 +22,9 @@ import { import {AGENT_NAME_MAX} from '@proxy/integrations'; import type {AuthenticatedRequest} from '../server/middleware/require-auth'; -import {manages} from '../utils/manages'; import {type OrgViewer, requireViewer} from '../utils/user-org'; import {ensureBuiltInAgent} from './built-in-agent'; +import {managesPersona, type PersonaResponse, toPersona} from './persona-response'; /** * The Chat page's routes, under the organization: its personas, which every member sees, each @@ -37,32 +36,12 @@ import {ensureBuiltInAgent} from './built-in-agent'; const MAX_MESSAGE = 20_000; -/** A persona as the page lists it, whether the viewer may change it, and the last thing said in their conversation with it, cut to a line; null before anything was said. */ -export type PersonaResponse = {id: string; name: string; look: Look; color: string; mayChange: boolean; lastMessage: {kind: 'person' | 'assistant'; text: string; at: string} | null}; - /** A turn of the conversation as the page shows it: what the person said, or what the persona answered, which is what went wrong where its turn failed. What the persona did in between is in the activity log, not here. */ export type EntryResponse = {id: string; kind: 'person' | 'assistant'; text: string; isError: boolean; at: string}; /** The member's conversation with a persona: the persona, its turns, and what the persona is up to: busy on it, asleep until a time, or waiting for the person. */ export type ThreadLogResponse = {persona: PersonaResponse; entries: EntryResponse[]; busy: boolean; sleepingUntil: string | null}; -// Whether the viewer may rename, reface or delete the persona: whoever made it, or the organization's owner (utils/manages.ts). -function managesPersona(viewer: OrgViewer, persona: PersonaRow): boolean { - return manages(viewer, {ownerUserId: null, byId: persona.createdById}); -} - -function toPersona(viewer: OrgViewer, persona: PersonaRow & {lastMessage?: LastMessage | null}): PersonaResponse { - const last = persona.lastMessage ?? null; - return { - id: persona.id, - name: persona.name, - look: persona.look, - color: persona.color, - mayChange: managesPersona(viewer, persona), - lastMessage: last && {kind: last.kind, text: preview(last.text), at: last.createdAt.toISOString()}, - }; -} - // A persona as the person makes or changes it: a name, and a head, a look in one of the colors. const personaSchema = z.object({ name: z.string().trim().pipe(z.string().min(1).max(AGENT_NAME_MAX)), diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index 9b345794..c4707b73 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -46,6 +46,7 @@ import { handleShareConnectionRoute, } from '../connections/route'; import {handleFaviconRoute} from '../favicons/route'; +import {handleInspectRequestRoute, handleInspectThreadRoute} from '../harness/inspect-route'; import {handleCreatePersonaRoute, handleDeletePersonaRoute, handleGetThreadRoute, handleListPersonasRoute, handlePostMessageRoute, handleUpdatePersonaRoute} from '../harness/route'; import {handleFillIntentRoute, handlePickIntentRoute} from '../intent/route'; import {handleMcpMethodNotAllowedRoute, handleMcpRoute} from '../mcp/route'; @@ -219,6 +220,10 @@ export function createApiRouter(): express.Router { org.delete('/chat/personas/:personaId', withAuthResult('Delete persona route', handleDeletePersonaRoute)); org.get('/chat/personas/:personaId/thread', withAuthResult('Get chat thread route', handleGetThreadRoute)); org.post('/chat/personas/:personaId/messages', withAuthResult('Post chat message route', handlePostMessageRoute)); + // The inspector, behind the `debug` experiment too: the conversation as the harness has + // it, and what the model was sent for an answer. + org.get('/chat/personas/:personaId/inspect', withAuthResult('Inspect chat thread route', handleInspectThreadRoute)); + org.get('/chat/inspect/entries/:entryId/request', withAuthResult('Inspect chat request route', handleInspectRequestRoute)); // The agent side: its own sign-in and cookie, and every request made for one of the agent's // tasks, checked against the agent's access and logged with the task. diff --git a/packages/db/src/harness.test.ts b/packages/db/src/harness.test.ts index db2eaf3c..3cfa913b 100644 --- a/packages/db/src/harness.test.ts +++ b/packages/db/src/harness.test.ts @@ -118,7 +118,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['assistant', job.id, 'You are signed in.'], ['tool_result', job.id, null], ]); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect(events.map((event) => [event.jobId, event.kind, event.workerId, event.error])).toEqual([ [job.id, 'claimed', 'worker-a', null], [job.id, 'done', 'worker-a', null], @@ -168,7 +168,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { ['tool_result', tool.id, false], ['assistant', next.id, true], ]); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect( events.slice(2).map((event) => [event.jobId, event.kind, event.workerId, event.error]), ).toEqual([ @@ -188,7 +188,7 @@ describe.skipIf(!databaseUrl)('the harness, against the database', () => { throw new Error('nothing to claim'); } unwrap(await finishJob(job.id, 'worker-e')); - const events = unwrap(await listJobEvents(thread.id)); + const events = unwrap(await listJobEvents([thread.id])); expect(events.slice(-2).map((event) => [event.jobId, event.kind, event.workerId])).toEqual([ [job.id, 'claimed', 'worker-e'], [job.id, 'done', 'worker-e'], diff --git a/packages/db/src/harness.ts b/packages/db/src/harness.ts index 0a313d25..71fe93fc 100644 --- a/packages/db/src/harness.ts +++ b/packages/db/src/harness.ts @@ -453,17 +453,67 @@ export async function listOpenJobs(threadId: string): Promise> { +/** What happened to the threads' jobs, in order: every claim, requeue, finish and failure. */ +export async function listJobEvents(threadIds: string[]): Promise> { return wrapDb(() => db.agentJobEvent.findMany({ - where: {threadId}, + where: {threadId: {in: threadIds}}, select: jobEventSelect, orderBy: {at: 'asc'}, }), ); } +/** A job with its whole record, for the inspector: beside what the worker reads, why it failed, its lease and when it was made. */ +export type JobDetailRow = JobRow & { + leaseExpiresAt: Date | null; + error: string | null; + createdAt: Date; +}; + +const jobDetailSelect = {...jobSelect, leaseExpiresAt: true, error: true, createdAt: true} as const; + +/** Every job of the threads, whatever its status, oldest first: the queue and its history, for the inspector. */ +export async function listThreadJobs( + threadIds: string[], +): Promise> { + return wrapDb(() => + db.agentJob.findMany({ + where: {threadId: {in: threadIds}}, + select: jobDetailSelect, + orderBy: {createdAt: 'asc'}, + }), + ); +} + +/** The sub-agents started from the thread, done or not, oldest first. */ +export async function listSubAgentThreads( + parentThreadId: string, +): Promise> { + return wrapDb(() => + db.agentThread.findMany({ + where: {parentThreadId}, + select: threadSelect, + orderBy: {createdAt: 'asc'}, + }), + ); +} + +/** The thread's whole log, for the inspector: what the model reads in order, then what is pending, in the order it landed. */ +export async function listEntriesWithPending( + threadId: string, +): Promise> { + return ( + await wrapDb(() => + db.agentEntry.findMany({ + where: {threadId}, + select: entrySelect, + orderBy: [{pending: 'asc'}, {seq: 'asc'}], + }), + ) + ).map((entries) => entries.map(toEntry)); +} + /** A new task for a sub-agent's calls, once the one before was abandoned. */ export async function setThreadTask( threadId: string, diff --git a/packages/utils/src/experiments.ts b/packages/utils/src/experiments.ts index 2b5c326a..11062aec 100644 --- a/packages/utils/src/experiments.ts +++ b/packages/utils/src/experiments.ts @@ -11,6 +11,9 @@ export const EXPERIMENTS = { // The built-in agent: a conversation with an agent Personal Agent Proxy runs itself, behind its // own gate, on the Chat page. harness: 'Built-in agent', + // Debugging: the inspector of the built-in agent, for whoever works on it, every thread of a + // conversation with its whole log, its jobs and what the model was sent, on its own page. + debug: 'Debugging', } as const satisfies Record; export type Experiment = keyof typeof EXPERIMENTS; diff --git a/packages/web/src/client/harness-client.ts b/packages/web/src/client/harness-client.ts index 70e3c66e..6939c6b9 100644 --- a/packages/web/src/client/harness-client.ts +++ b/packages/web/src/client/harness-client.ts @@ -12,7 +12,7 @@ const personaSchema = z.object({ color: z.enum(AVATAR_COLORS), }); -const chatPersonaSchema = personaSchema.extend({ +export const chatPersonaSchema = personaSchema.extend({ id: z.string(), // Whether the person may rename, reface or delete it: they made it, or own the organization. mayChange: z.boolean(), diff --git a/packages/web/src/client/inspect-client.ts b/packages/web/src/client/inspect-client.ts new file mode 100644 index 00000000..e287fc2c --- /dev/null +++ b/packages/web/src/client/inspect-client.ts @@ -0,0 +1,148 @@ +import {z} from 'zod'; +import {apiRequest, orgApi} from '@/client/request'; +import {chatPersonaSchema} from '@/client/harness-client'; + +// The inspector's api: the person's conversation with a persona as the harness has it, every +// thread of it with its whole log, its jobs and what happened to each; and, for an answer of the +// model, the request it was sent. + +const toolCallSchema = z.object({id: z.string(), name: z.string(), arguments: z.string()}); + +export const entryKinds = ['person', 'assistant', 'tool_result', 'event', 'summary'] as const; + +// An entry of a thread's log, whole: every kind, what is pending too. +const inspectedEntrySchema = z.object({ + id: z.string(), + seq: z.number(), + kind: z.enum(entryKinds), + text: z.string(), + toolCalls: z.array(toolCallSchema).nullable(), + callId: z.string().nullable(), + isError: z.boolean(), + // Landed while the model was answering: not in the log it reads until the answer is. + pending: z.boolean(), + inputTokens: z.number().nullable(), + outputTokens: z.number().nullable(), + // A summary's reach: the seq of the last entry it stands for. + coversSeq: z.number().nullable(), + // Whether the answer kept the system prompt it was sent, which the request route gives. + hasSystem: z.boolean(), + // The job that wrote the entry; null for what the person wrote. + jobId: z.string().nullable(), + at: z.string(), +}); + +export const jobStatuses = ['queued', 'running', 'done', 'failed'] as const; + +// A job as the queue has it, whatever its status. +const inspectedJobSchema = z.object({ + id: z.string(), + threadId: z.string(), + kind: z.enum(['inference', 'tool']), + status: z.enum(jobStatuses), + // A tool job's call: the entry it is in and the id the model gave it. + entryId: z.string().nullable(), + callId: z.string().nullable(), + runAt: z.string(), + claimedBy: z.string().nullable(), + leaseExpiresAt: z.string().nullable(), + attempts: z.number(), + error: z.string().nullable(), + createdAt: z.string(), +}); + +// What happened to a job, one row per happening. +const inspectedJobEventSchema = z.object({ + id: z.string(), + jobId: z.string(), + threadId: z.string(), + kind: z.enum(['claimed', 'requeued', 'done', 'failed']), + workerId: z.string().nullable(), + error: z.string().nullable(), + at: z.string(), +}); + +// Where the thread stands for the harness. +const threadStateSchema = z.object({ + ready: z.boolean(), + pendingCalls: z.array(toolCallSchema), + turnsAlone: z.number(), + maxTurnsAlone: z.number(), + lastInputTokens: z.number(), + compactAtTokens: z.number(), + compactionCoversSeq: z.number().nullable(), +}); + +const inspectedThreadSchema = z.object({ + id: z.string(), + parentThreadId: z.string().nullable(), + parentCallId: z.string().nullable(), + title: z.string().nullable(), + taskId: z.string().nullable(), + finishedAt: z.string().nullable(), + createdAt: z.string(), + entries: z.array(inspectedEntrySchema), + jobs: z.array(inspectedJobSchema), + events: z.array(inspectedJobEventSchema), + state: threadStateSchema, +}); + +const inspectionSchema = z.object({ + persona: chatPersonaSchema, + now: z.string(), + // The conversation first, then every sub-agent started from it, oldest first. + threads: z.array(inspectedThreadSchema), +}); + +// The conversation as the model reads it, in the chat completions form. +const modelMessageSchema = z.discriminatedUnion('role', [ + z.object({role: z.literal('system'), content: z.string()}), + z.object({role: z.literal('user'), content: z.string()}), + z.object({ + role: z.literal('assistant'), + content: z.string().nullable(), + tool_calls: z + .array( + z.object({ + id: z.string(), + type: z.literal('function'), + function: z.object({name: z.string(), arguments: z.string()}), + }), + ) + .optional(), + }), + z.object({role: z.literal('tool'), tool_call_id: z.string(), content: z.string()}), +]); + +const modelToolSchema = z.object({ + name: z.string(), + description: z.string(), + parameters: z.unknown(), +}); + +const requestSchema = z.object({ + kind: z.enum(['turn', 'summary']), + // Null where the answer was written without the model. + system: z.string().nullable(), + messages: z.array(modelMessageSchema), + tools: z.array(modelToolSchema), +}); + +export type ToolCall = z.infer; +export type InspectedEntry = z.infer; +export type InspectedJob = z.infer; +export type InspectedJobEvent = z.infer; +export type ThreadState = z.infer; +export type InspectedThread = z.infer; +export type Inspection = z.infer; +export type ModelMessage = z.infer; +export type ModelTool = z.infer; +export type ModelRequest = z.infer; + +export function getInspection(org: string, personaId: string) { + return apiRequest('GET', `${orgApi(org)}/chat/personas/${personaId}/inspect`, inspectionSchema); +} + +export function getRequest(org: string, entryId: string) { + return apiRequest('GET', `${orgApi(org)}/chat/inspect/entries/${entryId}/request`, requestSchema); +} diff --git a/packages/web/src/components/inspect-log.tsx b/packages/web/src/components/inspect-log.tsx new file mode 100644 index 00000000..4c55b690 --- /dev/null +++ b/packages/web/src/components/inspect-log.tsx @@ -0,0 +1,196 @@ +import {Badge} from '@proxy/ui/components/badge'; +import {Button} from '@proxy/ui/components/button'; +import {cn} from '@proxy/ui/lib/utils'; +import type {InspectedEntry, InspectedThread, ThreadState} from '@/client/inspect-client'; +import {formatTime} from '@/lib/format'; +import {coveredSeq, prettyJson} from '@/lib/inspect'; + +// The inspector's middle: a thread's whole log as the harness has it, every entry with what the +// model doesn't see (its seq, its cost, its job), and above it where the thread stands for the +// harness: whether the model is due, what holds it, how close the log is to being summarized. + +// Text longer than this is folded, with its length shown. +const FOLD_AT = 600; + +const KIND_CLASS: Record = { + person: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + tool_result: 'bg-muted text-muted-foreground', + event: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', + summary: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', +}; + +function Folded({text, open = false}: {text: string; open?: boolean}) { + if (text.length <= FOLD_AT) { + return 
{text}
; + } + return ( +
+ + {text.length.toLocaleString('en-US')} characters + + 
{text}
+
+ ); +} + +function Calls({entry}: {entry: InspectedEntry}) { + const calls = entry.toolCalls ?? []; + if (calls.length === 0) { + return null; + } + return ( +
    + {calls.map((call) => ( +
  • +
    + {call.name} + {call.id} +
    + 
    +						{prettyJson(call.arguments)}
+
    +
    • + ))} +
+ ); +} + +// What an entry says beside its kind: which call it answers, how far a summary reaches. +function lead(entry: InspectedEntry): string | null { + if (entry.kind === 'tool_result') { + return `answers ${entry.callId ?? '?'}`; + } + if (entry.kind === 'event') { + return `report for ${entry.callId ?? '?'}`; + } + if (entry.kind === 'summary') { + return `covers up to #${entry.coversSeq ?? '?'}`; + } + return null; +} + +function EntryCard({ + entry, + covered, + selected, + onSelect, +}: { + entry: InspectedEntry; + covered: boolean; + selected: boolean; + onSelect: (entryId: string) => void; +}) { + const askable = entry.kind === 'assistant' || entry.kind === 'summary'; + const tokens = entry.inputTokens !== null || entry.outputTokens !== null; + return ( +
  • +
    + #{entry.seq} + + {entry.kind} + + {lead(entry) && {lead(entry)}} + {formatTime(entry.at)} + {tokens && ( + + {(entry.inputTokens ?? 0).toLocaleString('en-US')} in /{' '} + {(entry.outputTokens ?? 0).toLocaleString('en-US')} out + + )} + {entry.pending && pending} + {entry.isError && error} + {covered && summarized} + {entry.jobId !== null && job {entry.jobId.slice(-6)}} + {askable && ( + + )} +
    + {entry.text !== '' && } + +
    • + ); +} + +// Where the thread stands for the harness. +function StateStrip({state, pending}: {state: ThreadState; pending: number}) { + const share = Math.min(1, state.lastInputTokens / state.compactAtTokens); + const holds = + state.pendingCalls.length > 0 + ? `waiting on ${state.pendingCalls.map((call) => call.name).join(', ')}` + : state.ready + ? 'model due' + : 'nothing new for the model'; + return ( +
    + {holds} + {pending > 0 && {pending} pending} + + {state.turnsAlone}/{state.maxTurnsAlone} steps alone + + + + {state.lastInputTokens.toLocaleString('en-US')}/ + {state.compactAtTokens.toLocaleString('en-US')} tokens + + + + + + {state.compactionCoversSeq !== null && ( + next turn summarizes up to #{state.compactionCoversSeq} + )} +
    + ); +} + +export function ThreadLog({ + thread, + selectedEntryId, + onSelectEntry, +}: { + thread: InspectedThread; + selectedEntryId: string | null; + onSelectEntry: (entryId: string) => void; +}) { + const covered = coveredSeq(thread.entries); + const pending = thread.entries.filter((entry) => entry.pending).length; + return ( + <> + +
    +
      + {thread.entries.length === 0 && ( +
    • + Nothing in the log yet. +
      • + )} + {thread.entries.map((entry) => ( + + ))} +
    +
    + + ); +} diff --git a/packages/web/src/components/inspect-request.tsx b/packages/web/src/components/inspect-request.tsx new file mode 100644 index 00000000..a580f038 --- /dev/null +++ b/packages/web/src/components/inspect-request.tsx @@ -0,0 +1,207 @@ +import {useEffect, useState} from 'react'; +import {Badge} from '@proxy/ui/components/badge'; +import {cn} from '@proxy/ui/lib/utils'; +import { + getRequest, + type InspectedEntry, + type InspectedThread, + type ModelMessage, + type ModelRequest, +} from '@/client/inspect-client'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import {formatDuration, prettyJson, timedEvents} from '@/lib/inspect'; + +// The inspector's right column: what the model was sent for the answer picked in the log, as the +// api rebuilds it, the system prompt, the messages and the tools; what came back; and what +// happened to the job that ran it. + +const ROLE_CLASS: Record = { + system: 'bg-muted text-muted-foreground', + user: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + assistant: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + tool: 'bg-muted text-muted-foreground', +}; + +function Section({title, children}: {title: string; children: React.ReactNode}) { + return ( +
    +

    {title}

    + {children} +
    + ); +} + +function MessageCard({message}: {message: ModelMessage}) { + const calls = message.role === 'assistant' ? (message.tool_calls ?? []) : []; + const content = message.role === 'tool' ? message.content : (message.content ?? ''); + return ( +
  • +
    + + {message.role} + + {message.role === 'tool' && {message.tool_call_id}} + {content.length.toLocaleString('en-US')} chars +
    + {content !== '' && ( +
    + + {content.slice(0, 80).replace(/\s+/g, ' ')}… + + 
    {content}
    +
    + )} + {calls.map((call) => ( +
    +
    + {call.function.name} + {call.id} +
    + 
    +						{prettyJson(call.function.arguments)}
+
    +
    + ))} +
    • + ); +} + +// What happened to the job that wrote the entry: each claim, requeue, finish and failure. +function JobHistory({entry, thread}: {entry: InspectedEntry; thread: InspectedThread}) { + const job = thread.jobs.find((candidate) => candidate.id === entry.jobId); + const events = thread.events.filter((event) => event.jobId === entry.jobId); + if (!job) { + return

    No job is kept for this entry.

    ; + } + return ( +
      +
    • + {job.id} · {job.kind} · {job.status} · {job.attempts}{' '} + {job.attempts === 1 ? 'attempt' : 'attempts'} +
      • + {timedEvents(events).map(({event, sinceClaimMs}) => ( +
    • + {formatTime(event.at)} + {event.kind} + {event.workerId !== null && ( + {event.workerId} + )} + {sinceClaimMs !== null && ( + after {formatDuration(sinceClaimMs)} + )} + {event.error !== null && {event.error}} +
      • + ))} +
    + ); +} + +export function RequestPane({ + entry, + thread, +}: { + entry: InspectedEntry | null; + thread: InspectedThread; +}) { + const org = useOrgSlug(); + const [request, setRequest] = useState<{ + entryId: string; + result: ModelRequest | null; + error: string | null; + } | null>(null); + + useEffect(() => { + if (!entry) { + return; + } + let stale = false; + void getRequest(org, entry.id).then((result) => { + if (stale) { + return; + } + setRequest({ + entryId: entry.id, + result: result.isOk() ? result.value : null, + error: result.isErr() ? 'The request could not be read.' : null, + }); + }); + return () => { + stale = true; + }; + }, [org, entry]); + + if (!entry) { + return ( +

    + Pick an answer or a summary in the log to see what the model was sent for it. +

    + ); + } + const loaded = request?.entryId === entry.id ? request : null; + return ( +
    +
    + {loaded === null &&

    Reading…

    } + {loaded?.error !== null && loaded?.error !== undefined && ( +

    {loaded.error}

    + )} + {loaded?.result && loaded.result.system === null && ( +

    + This answer was written without the model, so there was no request; the messages are the + log as it was. +

    + )} + {loaded?.result && ( +

    + {loaded.result.kind === 'summary' ? 'A summarization' : 'A turn'}:{' '} + {loaded.result.messages.length} messages, {loaded.result.tools.length} tools +

    + )} +
    + {loaded?.result && ( + <> +
    +
      + {loaded.result.messages.map((message, index) => ( + + ))} +
    +
    + {loaded.result.tools.length > 0 && ( +
    +
      + {loaded.result.tools.map((tool) => ( +
    • +
      + {tool.name} +

      {tool.description}

      + 
      +												{JSON.stringify(tool.parameters, null, 2)}
+
      +
      +
      • + ))} +
    +
    + )} + + )} +
    +

    + {(entry.inputTokens ?? 0).toLocaleString('en-US')} tokens in,{' '} + {(entry.outputTokens ?? 0).toLocaleString('en-US')} out + {entry.toolCalls && + entry.toolCalls.length > 0 && + `, ${entry.toolCalls.length} ${entry.toolCalls.length === 1 ? 'call' : 'calls'}`} +

    + {entry.text !== '' && ( + 
    {entry.text}
    + )} +
    +
    + +
    +
    + ); +} diff --git a/packages/web/src/components/inspect-threads.tsx b/packages/web/src/components/inspect-threads.tsx new file mode 100644 index 00000000..450885f8 --- /dev/null +++ b/packages/web/src/components/inspect-threads.tsx @@ -0,0 +1,250 @@ +import {Link} from 'react-router'; +import {Badge} from '@proxy/ui/components/badge'; +import {cn} from '@proxy/ui/lib/utils'; +import type {InspectedJob, InspectedThread} from '@/client/inspect-client'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import { + formatDue, + formatDuration, + jobTitle, + lastAttempt, + threadLabel, + threadStatus, + type ThreadStatus, + tokensOf, +} from '@/lib/inspect'; + +// The inspector's left column: the conversation's threads, the conversation first and the +// sub-agents started from it under it, each with where it stands; and under them the queue, +// every job of every thread still to run or running, with its history behind a toggle. + +const STATUS_LABEL: Record = { + working: 'working', + queued: 'queued', + asleep: 'asleep', + waiting: 'waiting', + reported: 'reported', + failed: 'failed', +}; + +const STATUS_CLASS: Record = { + working: 'bg-blue-500/15 text-blue-700 dark:text-blue-300', + queued: 'bg-amber-500/15 text-amber-700 dark:text-amber-300', + asleep: 'bg-violet-500/15 text-violet-700 dark:text-violet-300', + waiting: 'bg-muted text-muted-foreground', + reported: 'bg-emerald-500/15 text-emerald-700 dark:text-emerald-300', + failed: 'bg-destructive/10 text-destructive', +}; + +export function StatusBadge({status}: {status: ThreadStatus}) { + return ( + + {STATUS_LABEL[status.kind]} + {status.until !== null && ` until ${formatTime(status.until)}`} + + ); +} + +function ThreadRow({ + thread, + now, + current, + onSelect, +}: { + thread: InspectedThread; + now: string; + current: boolean; + onSelect: (threadId: string) => void; +}) { + const org = useOrgSlug(); + const tokens = tokensOf(thread.entries); + const sub = thread.parentThreadId !== null; + return ( +
  • + + {thread.taskId !== null && ( + + Task in the activity log + + )} +
    • + ); +} + +export function ThreadTree({ + threads, + now, + current, + onSelect, +}: { + threads: InspectedThread[]; + now: string; + current: string; + onSelect: (threadId: string) => void; +}) { + return ( +
    +

    + Threads +

    +
      + {threads.map((thread) => ( + + ))} +
    +
    + ); +} + +const JOB_CLASS: Record = { + running: STATUS_CLASS.working, + queued: STATUS_CLASS.queued, + done: STATUS_CLASS.reported, + failed: STATUS_CLASS.failed, +}; + +// Running first, then queued by when they are due, then what is over, newest first. +function order(a: InspectedJob, b: InspectedJob): number { + const rank = (job: InspectedJob) => + job.status === 'running' ? 0 : job.status === 'queued' ? 1 : 2; + if (rank(a) !== rank(b)) { + return rank(a) - rank(b); + } + if (a.status === 'queued') { + return a.runAt.localeCompare(b.runAt); + } + return b.createdAt.localeCompare(a.createdAt); +} + +// What a job is up to, in a line: due when, held by whom until when, or how long it ran. +function jobDetail(job: InspectedJob, thread: InspectedThread, now: string): string { + if (job.status === 'queued') { + return `${formatDue(job.runAt, now)}${job.attempts > 0 ? `, attempt ${job.attempts + 1}` : ''}`; + } + const attempt = lastAttempt(job, thread.events); + if (job.status === 'running') { + const lease = + job.leaseExpiresAt === null + ? '' + : `, lease ${formatDue(job.leaseExpiresAt, now).replace('in ', '')} left`; + const since = + attempt.claimedAt === null + ? '' + : `, ${formatDuration(new Date(now).getTime() - new Date(attempt.claimedAt).getTime())} so far`; + return `${job.claimedBy ?? 'claimed'}${lease}${since}, attempt ${job.attempts}`; + } + const ran = attempt.ranMs === null ? '' : `ran ${formatDuration(attempt.ranMs)}, `; + return `${ran}${job.attempts} ${job.attempts === 1 ? 'attempt' : 'attempts'}`; +} + +function JobRow({ + job, + thread, + now, + onSelectThread, +}: { + job: InspectedJob; + thread: InspectedThread; + now: string; + onSelectThread: (threadId: string) => void; +}) { + return ( +
  • + + + {job.status} + + {jobTitle(job, thread.entries)} + + + {jobDetail(job, thread, now)} + {job.error !== null && {job.error}} +
    • + ); +} + +export function JobQueue({ + threads, + now, + showHistory, + onToggleHistory, + onSelectThread, +}: { + threads: InspectedThread[]; + now: string; + showHistory: boolean; + onToggleHistory: () => void; + onSelectThread: (threadId: string) => void; +}) { + const jobs = threads + .flatMap((thread) => thread.jobs.map((job) => ({job, thread}))) + .filter(({job}) => showHistory || job.status === 'queued' || job.status === 'running') + .sort((a, b) => order(a.job, b.job)); + return ( +
    +

    + Jobs + +

    + {jobs.length === 0 && ( +

    Nothing queued or running.

    + )} +
      + {jobs.map(({job, thread}) => ( + + ))} +
    +
    + ); +} diff --git a/packages/web/src/lib/inspect.test.ts b/packages/web/src/lib/inspect.test.ts new file mode 100644 index 00000000..3ef630db --- /dev/null +++ b/packages/web/src/lib/inspect.test.ts @@ -0,0 +1,187 @@ +import {describe, expect, test} from 'bun:test'; +import type {InspectedEntry, InspectedJob, InspectedThread} from '@/client/inspect-client'; +import { + coveredSeq, + formatDue, + formatDuration, + jobTitle, + lastAttempt, + threadStatus, + tokensOf, +} from './inspect'; + +const now = '2026-10-21T10:00:00.000Z'; + +function job(extra: Partial): InspectedJob { + return { + id: 'job-1', + threadId: 'thread-1', + kind: 'inference', + status: 'queued', + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 0, + error: null, + createdAt: now, + ...extra, + }; +} + +function entry(extra: Partial): InspectedEntry { + return { + id: 'e1', + seq: 1, + kind: 'person', + text: '', + toolCalls: null, + callId: null, + isError: false, + pending: false, + inputTokens: null, + outputTokens: null, + coversSeq: null, + hasSystem: false, + jobId: null, + at: now, + ...extra, + }; +} + +function thread(extra: Partial): InspectedThread { + return { + id: 'thread-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, + entries: [], + jobs: [], + events: [], + state: { + ready: false, + pendingCalls: [], + turnsAlone: 0, + maxTurnsAlone: 30, + lastInputTokens: 0, + compactAtTokens: 80_000, + compactionCoversSeq: null, + }, + ...extra, + }; +} + +describe('threadStatus', () => { + test('is at work while a job runs, and queued while one is due', () => { + expect(threadStatus(thread({jobs: [job({status: 'running'})]}), now).kind).toBe('working'); + expect(threadStatus(thread({jobs: [job({status: 'queued'})]}), now).kind).toBe('queued'); + }); + + test('is asleep until the first of its jobs due later, and only when every open job is', () => { + const later = job({id: 'j2', kind: 'tool', runAt: '2026-10-21T10:05:00.000Z'}); + const sooner = job({id: 'j3', kind: 'tool', runAt: '2026-10-21T10:01:00.000Z'}); + expect(threadStatus(thread({jobs: [later, sooner]}), now)).toEqual({ + kind: 'asleep', + until: '2026-10-21T10:01:00.000Z', + }); + expect(threadStatus(thread({jobs: [later, job({status: 'queued'})]}), now).kind).toBe('queued'); + }); + + test('waits for the person with nothing open, and a finished sub-agent reported, or failed where a job did', () => { + expect(threadStatus(thread({jobs: [job({status: 'done'})]}), now).kind).toBe('waiting'); + expect(threadStatus(thread({finishedAt: now, jobs: [job({status: 'done'})]}), now).kind).toBe( + 'reported', + ); + expect(threadStatus(thread({finishedAt: now, jobs: [job({status: 'failed'})]}), now).kind).toBe( + 'failed', + ); + }); +}); + +describe('jobTitle', () => { + test('names a tool job by its call, found in the entry it is in', () => { + const entries = [ + entry({id: 'e2', kind: 'assistant', toolCalls: [{id: 'c1', name: 'read', arguments: '{}'}]}), + ]; + expect(jobTitle(job({kind: 'tool', entryId: 'e2', callId: 'c1'}), entries)).toBe('read'); + expect(jobTitle(job({kind: 'tool', entryId: 'e9', callId: 'c1'}), entries)).toBe('tool'); + expect(jobTitle(job({}), entries)).toBe('inference'); + }); +}); + +describe('the log’s sums', () => { + test('add up what the answers cost, and say how far the latest summary reaches', () => { + const entries = [ + entry({id: 'e1', kind: 'assistant', inputTokens: 100, outputTokens: 10}), + entry({id: 'e2', seq: 2, kind: 'summary', coversSeq: 1, inputTokens: 50, outputTokens: 5}), + entry({id: 'e3', seq: 3, kind: 'assistant', inputTokens: 20, outputTokens: 2}), + ]; + expect(tokensOf(entries)).toEqual({input: 170, output: 17}); + expect(coveredSeq(entries)).toBe(1); + expect(coveredSeq([])).toBe(-1); + }); +}); + +describe('the times', () => { + test('are formatted to what matters at their size', () => { + expect(formatDuration(340)).toBe('340ms'); + expect(formatDuration(1234)).toBe('1.2s'); + expect(formatDuration(130_000)).toBe('2m 10s'); + expect(formatDue('2026-10-21T10:00:12.000Z', now)).toBe('in 12.0s'); + expect(formatDue('2026-10-21T09:00:00.000Z', now)).toBe('due'); + }); + + test('give a job’s last attempt from its events: the claim, and how long it ran once it ended', () => { + const events = [ + { + id: 'ev1', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed' as const, + workerId: 'w1', + error: null, + at: '2026-10-21T10:00:00.000Z', + }, + { + id: 'ev2', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'requeued' as const, + workerId: 'w1', + error: 'down', + at: '2026-10-21T10:00:01.000Z', + }, + { + id: 'ev3', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed' as const, + workerId: 'w2', + error: null, + at: '2026-10-21T10:00:06.000Z', + }, + ]; + expect(lastAttempt(job({}), events)).toEqual({ + claimedAt: '2026-10-21T10:00:06.000Z', + ranMs: null, + }); + const done = { + id: 'ev4', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'done' as const, + workerId: 'w2', + error: null, + at: '2026-10-21T10:00:08.500Z', + }; + expect(lastAttempt(job({}), [...events, done])).toEqual({ + claimedAt: '2026-10-21T10:00:06.000Z', + ranMs: 2500, + }); + expect(lastAttempt(job({}), [])).toEqual({claimedAt: null, ranMs: null}); + }); +}); diff --git a/packages/web/src/lib/inspect.ts b/packages/web/src/lib/inspect.ts new file mode 100644 index 00000000..213fd873 --- /dev/null +++ b/packages/web/src/lib/inspect.ts @@ -0,0 +1,144 @@ +import type { + InspectedEntry, + InspectedJob, + InspectedJobEvent, + InspectedThread, +} from '@/client/inspect-client'; + +/** + * What the inspector says of a thread, a job and a log, worked out from what the api gives: + * where a thread stands, what a job is for and how long it took, what a summary covers. + */ + +/** Where a thread stands, in a word: at work, queued or asleep on a job, waiting for the person (a conversation), reported or failed (a sub-agent). */ +export type ThreadStatus = { + kind: 'working' | 'queued' | 'asleep' | 'waiting' | 'reported' | 'failed'; + // Until when, for one asleep. + until: string | null; +}; + +const SLEEP_SOON_MS = 1000; + +export function threadStatus(thread: InspectedThread, now: string): ThreadStatus { + if (thread.finishedAt !== null) { + const failed = thread.jobs.some((job) => job.status === 'failed'); + return {kind: failed ? 'failed' : 'reported', until: null}; + } + const open = thread.jobs.filter((job) => job.status === 'queued' || job.status === 'running'); + if (open.some((job) => job.status === 'running')) { + return {kind: 'working', until: null}; + } + if (open.length === 0) { + return {kind: 'waiting', until: null}; + } + const time = new Date(now).getTime(); + // A queued job due later is a sleep; one due now is about to be claimed. + const later = open.filter((job) => new Date(job.runAt).getTime() - time > SLEEP_SOON_MS); + if (later.length === open.length) { + const first = [...later].sort((a, b) => a.runAt.localeCompare(b.runAt))[0]; + return {kind: 'asleep', until: first?.runAt ?? null}; + } + return {kind: 'queued', until: null}; +} + +/** What a thread is called in a list: the conversation, or a sub-agent by its title. */ +export function threadLabel(thread: InspectedThread): string { + return thread.parentThreadId === null ? 'Conversation' : (thread.title ?? 'Sub-agent'); +} + +/** Each of a job's events with how long after the attempt's claim it came; null for a claim, and before any. */ +export function timedEvents( + events: InspectedJobEvent[], +): Array<{event: InspectedJobEvent; sinceClaimMs: number | null}> { + let claimedAt: string | null = null; + return events.map((event) => { + const since = + event.kind !== 'claimed' && claimedAt !== null + ? new Date(event.at).getTime() - new Date(claimedAt).getTime() + : null; + if (event.kind === 'claimed') { + claimedAt = event.at; + } + return {event, sinceClaimMs: since}; + }); +} + +/** What a job is for: the model's turn, or the call it runs, by name, found in the entry the call is in. */ +export function jobTitle(job: InspectedJob, entries: InspectedEntry[]): string { + if (job.kind === 'inference') { + return 'inference'; + } + const entry = entries.find((candidate) => candidate.id === job.entryId); + const call = entry?.toolCalls?.find((candidate) => candidate.id === job.callId); + return call ? call.name : 'tool'; +} + +/** The sum of what the model's answers in the log cost. */ +export function tokensOf(entries: InspectedEntry[]): {input: number; output: number} { + return entries.reduce( + (sum, entry) => ({ + input: sum.input + (entry.inputTokens ?? 0), + output: sum.output + (entry.outputTokens ?? 0), + }), + {input: 0, output: 0}, + ); +} + +/** The seq of the last entry the log's latest summary stands for; -1 where there is none. */ +export function coveredSeq(entries: InspectedEntry[]): number { + return entries.findLast((entry) => entry.kind === 'summary')?.coversSeq ?? -1; +} + +/** "340ms", "1.2s", "2m 10s". */ +export function formatDuration(ms: number): string { + if (ms < 1000) { + return `${Math.max(0, Math.round(ms))}ms`; + } + if (ms < 60_000) { + return `${(ms / 1000).toFixed(1)}s`; + } + const minutes = Math.floor(ms / 60_000); + const seconds = Math.round((ms % 60_000) / 1000); + return `${minutes}m ${seconds}s`; +} + +/** "in 12s", "in 3m 0s"; "due" once the time has come. */ +export function formatDue(runAt: string, now: string): string { + const ms = new Date(runAt).getTime() - new Date(now).getTime(); + if (ms <= 0) { + return 'due'; + } + return `in ${formatDuration(ms)}`; +} + +/** The job's last attempt, from its events: when it was claimed, and how long it ran, where it ended. */ +export function lastAttempt( + job: InspectedJob, + events: InspectedJobEvent[], +): {claimedAt: string | null; ranMs: number | null} { + const own = events.filter((event) => event.jobId === job.id); + const claimed = own.findLast((event) => event.kind === 'claimed') ?? null; + const ended = + own.findLast( + (event) => event.kind !== 'claimed' && (claimed === null || event.at >= claimed.at), + ) ?? null; + if (!claimed) { + return {claimedAt: null, ranMs: null}; + } + if (!ended) { + return {claimedAt: claimed.at, ranMs: null}; + } + return { + claimedAt: claimed.at, + ranMs: new Date(ended.at).getTime() - new Date(claimed.at).getTime(), + }; +} + +/** The arguments pretty-printed where they are JSON, as written otherwise. */ +export function prettyJson(text: string): string { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } +} diff --git a/packages/web/src/loaders.ts b/packages/web/src/loaders.ts index f488e1c4..79ebd38a 100644 --- a/packages/web/src/loaders.ts +++ b/packages/web/src/loaders.ts @@ -10,6 +10,7 @@ import { } from '@/client/agents-client'; import {getConnection, listConnections} from '@/client/connections-client'; import {getThread, listPersonas} from '@/client/harness-client'; +import {getInspection} from '@/client/inspect-client'; import {getMe} from '@/client/me-client'; import {getAuthorizationRequest} from '@/client/oauth-client'; import {getInvitation, getOrg, listMembers, listOrgs} from '@/client/orgs-client'; @@ -132,6 +133,30 @@ export async function chatLoader(args: OrgParams & {params: {id?: string}}) { return {log: unwrapLoaderResult(log), personas: unwrapLoaderResult(personas).personas}; } +// The inspector of the conversation with a persona: the first persona's where the address names none. +export async function inspectIndexLoader(args: OrgParams) { + const org = orgOf(args); + const {personas} = unwrapLoaderResult(await listPersonas(org)); + const first = personas[0]; + if (!first) { + throw redirect(`/${org}`); + } + throw redirect(`/${org}/inspect/${first.id}`); +} + +// The conversation with a persona as the harness has it, and the personas to switch between. +export async function inspectLoader(args: OrgParams & {params: {id?: string}}) { + const org = orgOf(args); + const [inspection, personas] = await Promise.all([ + getInspection(org, args.params.id ?? ''), + listPersonas(org), + ]); + return { + inspection: unwrapLoaderResult(inspection), + personas: unwrapLoaderResult(personas).personas, + }; +} + // The vault is the one built-in connection, which every organization has twice: the person's own // there and the organization's, shared with everyone in it. The entries of each are kept in // Personal Agent Proxy. diff --git a/packages/web/src/pages/inspect-page.test.tsx b/packages/web/src/pages/inspect-page.test.tsx new file mode 100644 index 00000000..4d7b8f51 --- /dev/null +++ b/packages/web/src/pages/inspect-page.test.tsx @@ -0,0 +1,329 @@ +/// + +import {afterEach, beforeEach, describe, expect, mock, test} from 'bun:test'; +import {cleanup, fireEvent, render, screen, waitFor, within} from '@testing-library/react'; +import {createMemoryRouter, RouterProvider} from 'react-router'; +import {TooltipProvider} from '@proxy/ui/components/tooltip'; +import {ThemeProvider} from '@proxy/ui/components/theme-provider'; +import {HumanSide} from '@/components/human-side'; +import type {ChatPersona} from '@/client/harness-client'; +import type { + InspectedEntry, + InspectedJob, + InspectedThread, + Inspection, + ModelRequest, +} from '@/client/inspect-client'; +import {InspectPage} from '@/pages/inspect-page'; +import {testHuman} from '@/test-org'; + +const now = '2026-10-21T10:00:00.000Z'; +const look = {nose: 0.48, noseOut: 0.35, eye: 27, eyeTheta: 55, eyeLat: 8, eyes: 2 as const}; +const persona: ChatPersona = { + id: 'persona-1', + name: 'Finance', + look, + color: 'blue', + mayChange: true, + lastMessage: null, +}; + +function entry( + seq: number, + kind: InspectedEntry['kind'], + text: string, + extra: Partial = {}, +): InspectedEntry { + return { + id: `e${seq}`, + seq, + kind, + text, + toolCalls: null, + callId: null, + isError: false, + pending: false, + inputTokens: null, + outputTokens: null, + coversSeq: null, + hasSystem: false, + jobId: null, + at: now, + ...extra, + }; +} + +function job(id: string, threadId: string, extra: Partial = {}): InspectedJob { + return { + id, + threadId, + kind: 'inference', + status: 'done', + entryId: null, + callId: null, + runAt: now, + claimedBy: null, + leaseExpiresAt: null, + attempts: 1, + error: null, + createdAt: now, + ...extra, + }; +} + +const state = { + ready: false, + pendingCalls: [], + turnsAlone: 1, + maxTurnsAlone: 30, + lastInputTokens: 1200, + compactAtTokens: 80_000, + compactionCoversSeq: null, +}; + +// The person asked for invoices, the agent delegated, and the sub-agent is reading. +const conversation: InspectedThread = { + id: 'thread-1', + parentThreadId: null, + parentCallId: null, + title: null, + taskId: null, + finishedAt: null, + createdAt: now, + entries: [ + entry(1, 'person', 'Find my invoices'), + entry(2, 'assistant', '', { + toolCalls: [ + { + id: 'c1', + name: 'delegate', + arguments: '{"title":"Find the invoices","brief":"Find them"}', + }, + ], + inputTokens: 1200, + outputTokens: 40, + hasSystem: true, + jobId: 'job-1', + }), + entry(3, 'tool_result', 'Started the sub-agent “Find the invoices”.', { + callId: 'c1', + jobId: 'job-2', + }), + entry(9, 'person', 'Thanks', {pending: true}), + ], + jobs: [ + job('job-1', 'thread-1'), + job('job-2', 'thread-1', {kind: 'tool', entryId: 'e2', callId: 'c1'}), + ], + events: [ + { + id: 'ev1', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:58.000Z', + }, + { + id: 'ev2', + jobId: 'job-1', + threadId: 'thread-1', + kind: 'done', + workerId: 'host-1', + error: null, + at: '2026-10-21T10:00:00.500Z', + }, + ], + state, +}; +const subAgent: InspectedThread = { + id: 'thread-2', + parentThreadId: 'thread-1', + parentCallId: 'c1', + title: 'Find the invoices', + taskId: 'task-2', + finishedAt: null, + createdAt: now, + entries: [ + entry(4, 'person', 'Find them'), + entry(5, 'assistant', '', { + toolCalls: [ + {id: 'c2', name: 'read', arguments: '{"connection":"gmail-1","tool":"list_emails"}'}, + ], + inputTokens: 800, + outputTokens: 30, + hasSystem: true, + jobId: 'job-3', + }), + ], + jobs: [ + job('job-3', 'thread-2'), + job('job-4', 'thread-2', { + kind: 'tool', + status: 'running', + entryId: 'e5', + callId: 'c2', + claimedBy: 'host-1', + leaseExpiresAt: '2026-10-21T10:02:00.000Z', + }), + ], + events: [ + { + id: 'ev4', + jobId: 'job-3', + threadId: 'thread-2', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:50.000Z', + }, + { + id: 'ev5', + jobId: 'job-3', + threadId: 'thread-2', + kind: 'done', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:53.000Z', + }, + { + id: 'ev3', + jobId: 'job-4', + threadId: 'thread-2', + kind: 'claimed', + workerId: 'host-1', + error: null, + at: '2026-10-21T09:59:59.000Z', + }, + ], + state: { + ...state, + pendingCalls: [{id: 'c2', name: 'read', arguments: '{}'}], + lastInputTokens: 800, + }, +}; + +const inspection: Inspection = {persona, now, threads: [conversation, subAgent]}; + +const request: ModelRequest = { + kind: 'turn', + system: 'You are signed in.', + messages: [ + {role: 'system', content: 'You are signed in.'}, + {role: 'user', content: 'Find my invoices'}, + ], + tools: [{name: 'delegate', description: 'Delegates', parameters: {type: 'object'}}], +}; + +const realFetch = globalThis.fetch; +const fetchMock = mock<(input: string, init?: RequestInit) => Promise>(); + +beforeEach(() => { + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => { + if (input.endsWith('/request')) { + return Promise.resolve(Response.json(request)); + } + if (input.endsWith('/inspect')) { + return Promise.resolve(Response.json(inspection)); + } + return Promise.resolve(Response.json({error: 'unexpected'}, {status: 500})); + }); + globalThis.fetch = fetchMock as unknown as typeof fetch; +}); + +afterEach(() => { + cleanup(); + globalThis.fetch = realFetch; +}); + +function renderInspectPage() { + const router = createMemoryRouter( + [ + { + id: 'human', + loader: () => testHuman(), + element: , + children: [ + { + path: '/:org/inspect/:id', + loader: () => ({inspection, personas: [persona]}), + element: , + }, + ], + }, + ], + {initialEntries: ['/acme/inspect/persona-1']}, + ); + return render( + + + + + , + ); +} + +describe('InspectPage', () => { + test('lists the conversation and its sub-agents with where each stands, and the jobs open on them', async () => { + renderInspectPage(); + const threads = await screen.findByRole('region', {name: 'Threads'}); + const rows = within(threads).getAllByRole('button'); + expect(rows[0]?.textContent).toContain('Conversation'); + expect(rows[0]?.textContent).toContain('waiting'); + expect(rows[1]?.textContent).toContain('Find the invoices'); + expect(rows[1]?.textContent).toContain('working'); + expect(rows[1]?.textContent).toContain('1 open calls'); + expect( + within(threads).getByRole('link', {name: 'Task in the activity log'}).getAttribute('href'), + ).toBe('/acme/activity/tasks/task-2'); + + const jobs = screen.getByRole('region', {name: 'Jobs'}); + expect(jobs.textContent).toContain('running'); + expect(jobs.textContent).toContain('read'); + expect(jobs.textContent).toContain('host-1'); + expect(jobs.textContent).not.toContain('inference'); + fireEvent.click(within(jobs).getByRole('button', {name: 'With history'})); + expect(jobs.textContent).toContain('inference'); + expect(jobs.textContent).toContain('ran 2.5s'); + }); + + test('shows the conversation’s whole log, the calls with their arguments and what is pending', async () => { + renderInspectPage(); + const log = await screen.findByRole('list', {name: 'Log'}); + // The log's own items, not the calls listed within one. + const items = within(log) + .getAllByRole('listitem') + .filter((item) => item.parentElement === log); + expect(items.map((item) => item.textContent)).toEqual([ + expect.stringContaining('#1person'), + expect.stringContaining('delegate'), + expect.stringContaining('answers c1'), + expect.stringContaining('pending'), + ]); + expect(items[1]?.textContent).toContain('"title": "Find the invoices"'); + expect(items[1]?.textContent).toContain('1,200 in / 40 out'); + expect(screen.getByText(/nothing new for the model/)).toBeDefined(); + expect(screen.getByText('1 pending')).toBeDefined(); + }); + + test('opens a sub-agent’s log, and the request behind an answer in it', async () => { + renderInspectPage(); + const threads = await screen.findByRole('region', {name: 'Threads'}); + fireEvent.click(within(threads).getByRole('button', {name: /Find the invoices/})); + expect(screen.getByText(/waiting on read/)).toBeDefined(); + const log = screen.getByRole('list', {name: 'Log'}); + expect(log.textContent).toContain('list_emails'); + + fireEvent.click(within(log).getByRole('button', {name: 'Request'})); + await waitFor(() => { + expect(screen.getByText(/A turn: 2 messages, 1 tools/)).toBeDefined(); + }); + expect(fetchMock.mock.calls[0]?.[0]).toBe('/api/orgs/acme/chat/inspect/entries/e5/request'); + expect(screen.getByText('Find my invoices', {selector: 'pre'})).toBeDefined(); + expect(screen.getByText('delegate', {selector: 'summary'})).toBeDefined(); + expect(screen.getByText('claimed')).toBeDefined(); + expect(screen.getByText('after 3.0s')).toBeDefined(); + }); +}); diff --git a/packages/web/src/pages/inspect-page.tsx b/packages/web/src/pages/inspect-page.tsx new file mode 100644 index 00000000..c114a09d --- /dev/null +++ b/packages/web/src/pages/inspect-page.tsx @@ -0,0 +1,217 @@ +import {ArrowUpIcon} from 'lucide-react'; +import {type FormEvent, type KeyboardEvent, useCallback, useEffect, useRef, useState} from 'react'; +import {Link, useLoaderData, useNavigate} from 'react-router'; +import {Button} from '@proxy/ui/components/button'; +import { + InputGroup, + InputGroupAddon, + InputGroupButton, + InputGroupTextarea, +} from '@proxy/ui/components/input-group'; +import {postMessage} from '@/client/harness-client'; +import {getInspection, type Inspection} from '@/client/inspect-client'; +import {Head} from '@/components/head'; +import {ThreadLog} from '@/components/inspect-log'; +import {RequestPane} from '@/components/inspect-request'; +import {JobQueue, StatusBadge, ThreadTree} from '@/components/inspect-threads'; +import {useOrgSlug} from '@/hooks/use-org'; +import {formatTime} from '@/lib/format'; +import {threadLabel, threadStatus} from '@/lib/inspect'; +import {describeFetchError} from '@/lib/loader-utils'; +import type {inspectLoader} from '@/loaders'; + +/** + * The inspector: the person's conversation with a persona as the harness has it, on a page of + * its own, for whoever works on the built-in agent. Nothing of the Chat page: the conversation's + * threads and the queue on the left, the thread picked in the middle with its whole log and an + * input to say something in the conversation, and on the right what the model was sent for the + * answer picked. Read again as the agent works, as the Chat page is. + */ + +const BUSY_POLL_MS = 1000; +const IDLE_POLL_MS = 4000; + +type Loaded = Awaited>; + +export function InspectPage() { + const loaded = useLoaderData(); + return ; +} + +function hasOpenJobs(inspection: Inspection): boolean { + return inspection.threads.some((thread) => + thread.jobs.some((job) => job.status === 'queued' || job.status === 'running'), + ); +} + +function Inspector({loaded}: {loaded: Loaded}) { + const org = useOrgSlug(); + const navigate = useNavigate(); + const [inspection, setInspection] = useState(loaded.inspection); + const [threadId, setThreadId] = useState(loaded.inspection.threads[0]?.id ?? ''); + const [entryId, setEntryId] = useState(null); + const [showHistory, setShowHistory] = useState(false); + const [text, setText] = useState(''); + const [sending, setSending] = useState(false); + const [error, setError] = useState(null); + const form = useRef(null); + const personaId = loaded.inspection.persona.id; + const busy = hasOpenJobs(inspection); + + const latestRead = useRef(0); + const read = useCallback(async () => { + latestRead.current += 1; + const own = latestRead.current; + const result = await getInspection(org, personaId); + if (own !== latestRead.current || result.isErr()) { + return; + } + setInspection(result.value); + }, [org, personaId]); + + useEffect(() => { + const timer = setInterval(() => void read(), busy ? BUSY_POLL_MS : IDLE_POLL_MS); + return () => { + clearInterval(timer); + }; + }, [read, busy]); + + const thread = + inspection.threads.find((candidate) => candidate.id === threadId) ?? inspection.threads[0]; + const entry = thread?.entries.find((candidate) => candidate.id === entryId) ?? null; + + const send = async (event: FormEvent) => { + event.preventDefault(); + const message = text.trim(); + if (message === '' || sending) { + return; + } + setSending(true); + setError(null); + const posted = await postMessage(org, personaId, message); + setSending(false); + if (posted.isErr()) { + setError(describeFetchError(posted.error)); + return; + } + setText(''); + await read(); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Enter' && !event.shiftKey) { + event.preventDefault(); + form.current?.requestSubmit(); + } + }; + + const persona = inspection.persona; + return ( +
    +
    + + Inspector + + + Open in Chat + + + + {busy ? 'live, every second' : 'idle, every 4 seconds'} + + {formatTime(inspection.now)} + +
    +
    + +
    + {thread && ( + <> +
    + {threadLabel(thread)} + + {thread.id} + {thread.parentCallId !== null && ( + + from {thread.parentCallId} + + )} +
    + + {thread.parentThreadId === null && ( +
    void send(event)} + className="flex shrink-0 flex-col gap-1 border-t p-3" + > + {error &&

    {error}

    } + + setText(event.target.value)} + onKeyDown={onKeyDown} + placeholder={`Message ${persona.name}`} + aria-label="Message" + className="min-h-9 p-2 text-sm" + rows={1} + /> + + + + + + +
    + )} + + )} +
    + +
    +
    + ); +} diff --git a/packages/web/src/router.tsx b/packages/web/src/router.tsx index 78f3b5ac..c0148ee2 100644 --- a/packages/web/src/router.tsx +++ b/packages/web/src/router.tsx @@ -27,6 +27,8 @@ import { connectionLoader, connectionsLoader, homeLoader, + inspectIndexLoader, + inspectLoader, inviteLoader, membersLoader, shouldRevalidateHome, @@ -59,6 +61,7 @@ import {ConnectAgentPage} from '@/pages/connect-agent-page'; import {ConnectionPage} from '@/pages/connection-page'; import {ConnectionsPage} from '@/pages/connections-page'; import {HomePage} from '@/pages/home-page'; +import {InspectPage} from '@/pages/inspect-page'; import {InvitePage} from '@/pages/invite-page'; import {LoginPage} from '@/pages/login-page'; import {MembersPage} from '@/pages/members-page'; @@ -269,6 +272,10 @@ export const router = createBrowserRouter([ // The built-in agents, behind the `harness` experiment: the api refuses it otherwise. {path: '/:org/chat', loader: chatIndexLoader, element: }, {path: '/:org/chat/:id', loader: chatLoader, element: }, + // The inspector of the built-in agent, behind the `debug` experiment too, on a + // page of its own, reached by its address alone. + {path: '/:org/inspect', loader: inspectIndexLoader, element: }, + {path: '/:org/inspect/:id', loader: inspectLoader, element: }, {path: '/:org/agents', loader: agentsLoader, element: }, {path: '/:org/agents/new', loader: agentsLoader, element: }, {path: '/:org/agents/:id', loader: agentLoader, element: },