Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,9 @@ A folder whose `agent.yaml` names a `plugins` list now fails to load with `plugi

## Bug Fixes

### A hosted tool call in a group channel names the person who wrote

In a channel an agent config answers through `chat_channels`, a call of a tool your backend hosts (a dispatch worker) named no user, so a backend acting for "the user who asked" (`tools.ToolCallContextFrom(ctx).User.ID`) got an empty id and had to refuse. Each call now names the writer of the message its turn answers, by user id and name: Anya's turn names Anya, and Ben's next one Ben, whether the config answers every message or only its mentions. A session opened with a user's token still names that user.
### A POST from a sandbox through a tunnel to an allowed domain no longer fails after a pause

The router's sandbox proxy closed a tunnel to an allowed domain after 90 s with no traffic. That is also how long a Go client keeps a pooled connection, and the proxy's clock starts first. So a POST that a Go client sent on a kept connection near that limit could fail with `EOF`. The proxy now keeps such a tunnel for 4 min, as it keeps a connection to the proxy, so the client closes it first. The sandbox's TLS forwarder waits as long for a silent proxy (#2161, AI-2346).
Expand Down
10 changes: 9 additions & 1 deletion acceleration/internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,12 @@ type Agent struct {
// lastParticipant is who the agent was last talking to, so a reply prompted by
// delegated work coming back is attributed to the person who is waiting for it.
lastParticipant stt.Participant
lastHeardAt time.Time
// lastWriter is who wrote the turn being answered in a channel of a type an agent config
// answers in (WithWriter), and zero for any other turn: wherever lastParticipant is set for a
// heard or new turn, lastWriter is cleared with it. Such a turn's lastParticipant is the
// typed caller, who names no user, so a tool call names the writer instead (requesterLocked).
lastWriter writer
lastHeardAt time.Time
// talkedAt is when the caller last said more than a murmur (takesFloor), which is what
// overtakes a check-in held for the next voice.
talkedAt time.Time
Expand Down Expand Up @@ -1870,6 +1875,7 @@ func (a *Agent) consumeSTT(participantID string, session *sttrouter.Session, flo
a.mu.Lock()
a.lastHeardAt = time.Now()
a.lastParticipant = typed.Participant
a.lastWriter = writer{}
a.mu.Unlock()
continue
}
Expand Down Expand Up @@ -1936,6 +1942,7 @@ func (a *Agent) transcribed(typed stt.Transcript, count func(stt.Transcript)) {
a.talkedAt = a.lastHeardAt
}
a.lastParticipant = typed.Participant
a.lastWriter = writer{}
a.mu.Unlock()
if typed.Interim {
// Words of a turn still going on, from a transcriber that will say when it
Expand Down Expand Up @@ -2477,6 +2484,7 @@ func (a *Agent) respondTurn(
a.sender = sender
a.scheduled = scheduled
a.lastParticipant = participant
a.lastWriter = listened.writer
if !listened.typed {
a.heardCaller = true
}
Expand Down
4 changes: 4 additions & 0 deletions acceleration/internal/agent/native.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ func (a *Agent) respondNative(ctx context.Context, text string, images []llm.Ima
a.history = append(a.history, llm.Message{Role: llm.User, Content: harness.Defang(text)})
a.heardText = text
a.lastParticipant = caller
a.lastWriter = writer{}
a.scheduled = routing.ScheduledTurn(ctx)
a.mu.Unlock()
a.turns.typedTurn()
Expand All @@ -106,6 +107,7 @@ func (a *Agent) hear(inbound InboundAudio) {
if a.bound.ID == "" {
a.bound = inbound.Participant
a.lastParticipant = inbound.Participant
a.lastWriter = writer{}
a.logger.Debug("bound the speech-to-speech session to a participant",
"participant", inbound.Participant.ID)
}
Expand Down Expand Up @@ -322,6 +324,7 @@ func (a *Agent) consumeSTS(p *pipeline, speech *stsrouter.Session, events <-chan
case sts.SpeechStarted:
a.mu.Lock()
a.lastParticipant = a.participantOr(typed.Participant)
a.lastWriter = writer{}
a.lastHeardAt = typed.At
a.mu.Unlock()

Expand Down Expand Up @@ -560,6 +563,7 @@ func (a *Agent) heardNative(transcript sts.InputTranscript) {
a.history = append(a.history, llm.Message{Role: llm.User, Content: harness.Defang(text)})
a.heardText = text
a.lastParticipant = participant
a.lastWriter = writer{}
a.lastHeardAt = time.Now()
a.scheduled = false
// SpeechStopped starts the wait. Transcription can finish after the reply, so it
Expand Down
6 changes: 5 additions & 1 deletion acceleration/internal/agent/requester.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@ func RequesterFrom(ctx context.Context) (Requester, bool) {
}

// requesterLocked is who the turn being answered came from. A participant heard on a call
// names their user; a typed command names its sender. The caller holds a.mu.
// names their user; a message written in a channel of a type an agent config answers in names
// its writer (WithWriter); a typed command names its sender. The caller holds a.mu.
func (a *Agent) requesterLocked() Requester {
requester := Requester{UserID: a.lastParticipant.UserID, Name: a.lastParticipant.Name}
if requester.UserID == "" {
requester = Requester{UserID: a.lastWriter.id, Name: a.lastWriter.name}
}
if requester.UserID == "" {
requester.UserID = a.sender.UserID
}
Expand Down
105 changes: 105 additions & 0 deletions acceleration/internal/agent/requester_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ package agent

import (
"context"
"fmt"
"time"

"github.com/GetStream/Vision-Agents/acceleration/internal/llm"
"github.com/GetStream/Vision-Agents/acceleration/internal/options"
"github.com/GetStream/Vision-Agents/acceleration/internal/routing"
"github.com/GetStream/Vision-Agents/acceleration/internal/stt"
)

Expand Down Expand Up @@ -36,3 +39,105 @@ func (s *AgentSuite) TestAToolAskedForInASpokenTurnIsForWhoeverSpokeIt() {
s.Fail("the tool never ran")
}
}

// AI-3464: in a channel of a type an agent config answers in, a tool call is for the person who
// wrote the message its turn answers (WithWriter), turn by turn: Anya's turn is hers, Ben's next
// one his, a turn nobody wrote there is nobody's, and a writer goes before the caller who relayed
// their message.
func (s *AgentSuite) TestAToolAskedForInAWrittenTurnIsForWhoeverWroteIt() {
Comment thread
kanat marked this conversation as resolved.
Resolved
s.ownsTools("")
s.joinText()
runner := &requesterTool{seen: make(chan Requester, 1)}
s.agent.options.ToolRunner = runner
s.asksFor("lookup_order", `{"order":"12"}`)
relayed := routing.WithCaller(s.ctx, routing.Caller{UserID: "backend-sam"})

s.Equal(Requester{UserID: "anya-1", Name: "Anya"}, s.toolAskedIn(WithWriter(s.ctx, "anya-1", "Anya"), runner))
s.Equal(Requester{UserID: "ben-2", Name: "Ben"}, s.toolAskedIn(WithWriter(s.ctx, "ben-2", "Ben"), runner), "Anya's turn named Ben's call")
s.Equal(Requester{}, s.toolAskedIn(s.ctx, runner), "a turn nobody wrote named the last writer")
s.Equal(Requester{UserID: "backend-sam"}, s.toolAskedIn(relayed, runner), "a caller's turn")
s.Equal(Requester{UserID: "anya-1", Name: "Anya"}, s.toolAskedIn(WithWriter(relayed, "anya-1", "Anya"), runner),
"a writer's message relayed by a caller")
}

// toolAskedIn has the agent answer a turn on ctx whose first reply calls the tool asksFor named,
// and is who that call was for, once the turn is answered. Each turn's call has an id of its own.
func (s *AgentSuite) toolAskedIn(ctx context.Context, runner *requesterTool) Requester {
answered := countOf[Responded](s.reported())
s.model.mu.Lock()
s.model.callsOn = len(s.model.asked) + 1
s.model.calls[0].ID = fmt.Sprintf("call-%d", s.model.callsOn)
s.model.mu.Unlock()
_, err := s.agent.RespondTo(ctx, "where is my order", nil, options.LLM{})
s.Require().NoError(err)
var requester Requester
select {
case requester = <-runner.seen:
case <-time.After(settleFor):
s.Fail("the tool never ran")
}
// One reply asks for the tool and the next one answers with its result.
s.eventually(func() bool { return countOf[Responded](s.reported()) == answered+2 }, "the turn was never answered")
return requester
}

// renamingTool is requesterTool that has the model's next call carry an id and arguments of its
// own, because a repeat of the same call is answered from the first instead of run again.
type renamingTool struct {
requesterTool
s *AgentSuite
}

func (r *renamingTool) Run(ctx context.Context, call llm.ToolCall) ([]llm.ContentPart, error) {
r.s.model.mu.Lock()
r.s.model.calls[0].ID = call.ID + "-next"
r.s.model.calls[0].Arguments = `{"order":"13"}`
r.s.model.mu.Unlock()
return r.requesterTool.Run(ctx, call)
}

// AI-3464: a writer's turn that calls a second tool, once the first one's result is in (look up,
// then act), is still the writer's: the later round is not a turn of its own.
func (s *AgentSuite) TestEveryToolRoundOfAWrittenTurnIsForWhoeverWroteIt() {
s.ownsTools("")
s.joinText()
runner := &renamingTool{requesterTool: requesterTool{seen: make(chan Requester, 2)}, s: s}
s.agent.options.ToolRunner = runner
s.asksFor("lookup_order", `{"order":"12"}`)
s.model.mu.Lock()
s.model.keepCalling, s.model.callsUntil = true, 3
s.model.mu.Unlock()
_, err := s.agent.RespondTo(WithWriter(s.ctx, "anya-1", "Anya"), "where is my order", nil, options.LLM{})
s.Require().NoError(err)
for round := 1; round <= 2; round++ {
select {
case requester := <-runner.seen:
s.Equal(Requester{UserID: "anya-1", Name: "Anya"}, requester, "round %d", round)
case <-time.After(settleFor):
s.Failf("the tool never ran", "round %d", round)
}
}
}

// AI-3464: a turn answered with a reply started for it is not the written turn before it, so its
// tool calls do not name that turn's writer.
func (s *AgentSuite) TestAnAdoptedSpeculationDoesNotKeepTheLastWriter() {
s.onFakeClock(func() {
s.speculates = true
s.ownsTools("")
s.join(false)
s.model.reply = []string{"It ships tomorrow morning."}
s.agent.mu.Lock()
s.agent.lastWriter = writer{id: "anya-1", name: "Anya"}
s.agent.mu.Unlock()
ready := candidate{ID: "next-question", Text: "What time tomorrow?", ReadyAt: time.Now()}

s.agent.speculate(s.agent.harness, ready, false, false, noOpinion)
s.Require().True(s.agent.adoptSpeculation(ready, ""))

s.agent.mu.Lock()
requester := s.agent.requesterLocked()
s.agent.mu.Unlock()
s.Equal(Requester{}, requester, "the heard turn named the writer of the turn before it")
})
}
1 change: 1 addition & 0 deletions acceleration/internal/agent/speculate.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ func (a *Agent) adoptSpeculation(ready candidate, note string) (adopted bool, sp
a.sender = routing.Caller{}
a.scheduled = false
a.lastParticipant = ready.Participant
a.lastWriter = writer{}
a.heardCaller = true
a.generatingCancel[ready.ID] = spec.cancel
a.mu.Unlock()
Expand Down
2 changes: 2 additions & 0 deletions acceleration/internal/api/app_hooks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func (s *AppHooksSuite) SetupSuite() {
s.approvalWait = 10 * time.Second
}
s.RouterSuite.SetupSuite()
// Sessions offer what the app's workers host, as cmd/router has them (chat_channels_hosted_test.go).
s.manager.HostTools(s.dispatch)
s.token = issuedToken(&s.RouterSuite, s.provider)
}

Expand Down
90 changes: 90 additions & 0 deletions acceleration/internal/api/chat_channels_hosted_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
//go:build integration

package api

import (
"context"
"fmt"
"net/http"
"time"

"github.com/gorilla/websocket"

"github.com/GetStream/Vision-Agents/acceleration/internal/dispatch"
"github.com/GetStream/Vision-Agents/acceleration/internal/store"
)

// AI-3464: in a channel of a type a config answers in, a call of a tool the app's backend hosts
// (a dispatch worker) names the person who wrote the message its turn answers, as a session
// opened with that person's token does, so the backend can act for them: Anya's turn names Anya,
// and Ben's next one Ben.

func (s *AppHooksSuite) TestAHostedCallInAChannelTypeAnsweringMentionsIsForEachTurnsWriter() {
s.hostedCallsAreForEachWriter(store.ChatChannelAnswerMentioned)
}

func (s *AppHooksSuite) TestAHostedCallInAChannelTypeAnsweringEveryMessageIsForEachTurnsWriter() {
s.hostedCallsAreForEachWriter(store.ChatChannelAnswerAlways)
}

// hostedCallsAreForEachWriter has Anya, then Ben, ask a config answering in callouts as answer
// says, on the model that reaches for lookup_order, which the backend's worker hosts for it and
// a tool group of the config names.
func (s *AppHooksSuite) hostedCallsAreForEachWriter(answer string) {
s.holdsType("callouts", memberWrites)
config := store.AgentConfig{CustomerID: s.customerID(), Name: "rota-" + s.utils.uuid(), Mode: store.AgentModeText,
LLM: "tooling/tool-model", ChatChannels: []store.ChatChannel{{Type: "callouts", Answer: answer}},
ToolGroups: []store.ToolGroup{{Name: "orders", Tools: []string{lookupOrder}}}}
s.Require().NoError(s.store.CreateAgentConfig(context.Background(), &config))
worker := s.hostsLookupOrder(config.ID)
mention := ""
if answer == store.ChatChannelAnswerMentioned {
mention = fmt.Sprintf(`[{"id": %q, "name": %q}]`, config.ID, config.Name)
}
anya, ben := s.person("Anya"), s.person("Ben")
channel := s.teamChannel("callouts", anya, ben)
cid := "callouts:" + channel

s.says("callouts", channel, anya, "where is my order 12?", mention)
s.Equal(map[string]any{"id": anya, "name": "Anya"}, s.hostedCallUser(worker), "Anya's turn")
s.Require().Eventually(func() bool { return s.replies(cid) == 1 }, settleFor, 20*time.Millisecond)

s.says("callouts", channel, ben, "and where is mine?", mention)
s.Equal(map[string]any{"id": ben, "name": "Ben"}, s.hostedCallUser(worker), "Ben's turn after Anya's")
s.Require().Eventually(func() bool { return s.replies(cid) == 2 }, settleFor, 20*time.Millisecond)
}

// hostsLookupOrder connects a worker of the app's backend hosting lookup_order for the agent id given.
func (s *RouterSuite) hostsLookupOrder(agentID string) *websocket.Conn {
worker := s.serverClient.opens("/v1/dispatch")
s.await(worker, "ready")
s.Require().NoError(s.dispatch.Host(s.dispatch.Workers(s.customerID())[0], agentID,
[]dispatch.Tool{{Name: lookupOrder, Description: "find an order by its number"}}, time.Minute))
return worker
}

// hostedCallUser is the user the next call the worker is handed names, nil when it names none,
// once the worker has answered it.
func (s *RouterSuite) hostedCallUser(worker *websocket.Conn) any {
call := s.await(worker, "tool_call")
s.Require().Equal(lookupOrder, call["name"])
s.Require().NoError(worker.WriteJSON(map[string]any{"type": "tool_result", "id": call["id"], "output": "it ships tomorrow"}))
return call["user"]
}

// The control: a session a person opened with their own token names them on a hosted call, as
// on base b466fb795 (probe of this test there).
func (s *SessionToolsSuite) TestAHostedCallInASessionOpenedWithAUsersTokenIsForThem() {
agent := "agent-" + s.utils.uuid()
worker := s.hostsLookupOrder(agent)
device := s.data.createUser()
request := textSession(nil)
request.AgentId, request.Llm = &agent, pointerTo("tooling/tool-model")
opened := device.createSession(request)

status, body := s.serverClient.actingFor(device).call(http.MethodPost,
"/v1/agents/sessions/"+opened.Id+"/respond", RespondRequest{Text: "where is my order", RequestId: pointerTo("request-" + s.utils.uuid())})
s.Require().Equal(http.StatusOK, status, string(body))

s.Equal(map[string]any{"id": device.userID}, s.hostedCallUser(worker))
}
Loading