Skip to content

feat(gooddata-eval): keep workflowStatus parts out of scored answers #1926

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tychtjan wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(gooddata-eval): keep workflowStatus parts out of scored answers #1926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
"searchResults",
"alertProposal",
"clarifyingQuestions",
"workflowStatus",
}
)

Expand Down Expand Up @@ -246,6 +247,9 @@ def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator, item_id: st
acc.alert_proposals.append(part.get("alertProposal") or {})
elif ptype == "searchResults":
acc.search_results.append(part)
elif ptype == "workflowStatus":
# A UI signal that the turn ended a workflow; it carries nothing the answer is scored on.
continue
else:
if ptype not in _KNOWN_PART_TYPES:
_log.warning("unknown multipart part type %r; captured as an unhandled part", ptype)
Expand Down
13 changes: 13 additions & 0 deletions packages/gooddata-eval/tests/test_chat_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,18 @@ def test_alert_proposal_part_still_goes_to_its_own_field():
assert result.unhandled_parts == []


def test_workflow_status_part_stays_out_of_the_answer():
result = parse_sse_lines(
_multipart_lines(
{"type": "workflowStatus", "skills": ["dashboard_editor"], "status": "ended"},
{"type": "text", "text": "Done."},
)
)
assert result.unhandled_parts == []
assert result.search_results == []
assert render_answer_text(result) == "Done."


def test_known_part_types_matches_the_documented_gen_ai_union():
gen_ai_message_part_union = {
"text",
Expand All @@ -118,6 +130,7 @@ def test_known_part_types_matches_the_documented_gen_ai_union():
"searchResults",
"alertProposal",
"clarifyingQuestions",
"workflowStatus",
}
assert gen_ai_message_part_union == _KNOWN_PART_TYPES

Expand Down
Loading