From e62283f1b6c05fd440641a5fbd8c5b314456dbb0 Mon Sep 17 00:00:00 2001 From: Jan Tychtl Date: Fri, 6 Nov 2026 15:29:25 +0100 Subject: [PATCH] feat(gooddata-eval): keep workflowStatus parts out of scored answers gen-ai is adding a stored workflowStatus multipart part that tells the browser a dashboard workflow ended with nothing left open. It rides on the turn's last visible answer, next to the text. The SSE client put every non-text part it did not handle into unhandled_parts, and render_answer_text appends those to the text the evaluators score, so dashboard evals would grade a JSON blob as part of the answer and log an unknown-part warning on every decline turn. Register the type and skip it in _handle_multipart: it carries nothing an evaluator scores. jira: GDAI-2733 risk: low --- .../src/gooddata_eval/core/chat/sse_client.py | 4 ++++ packages/gooddata-eval/tests/test_chat_render.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index ce7ead5c1..b8389c056 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -71,6 +71,7 @@ "searchResults", "alertProposal", "clarifyingQuestions", + "workflowStatus", } ) @@ -246,6 +247,9 @@ def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator, item_id: st acc.alert_proposals.append(part.get("alertProposal") or {}) elif ptype == "searchResults": acc.search_results.append(part) + elif ptype == "workflowStatus": + # A UI signal that the turn ended a workflow; it carries nothing the answer is scored on. + continue else: if ptype not in _KNOWN_PART_TYPES: _log.warning("unknown multipart part type %r; captured as an unhandled part", ptype) diff --git a/packages/gooddata-eval/tests/test_chat_render.py b/packages/gooddata-eval/tests/test_chat_render.py index 29904158f..ef7d7bdbb 100644 --- a/packages/gooddata-eval/tests/test_chat_render.py +++ b/packages/gooddata-eval/tests/test_chat_render.py @@ -106,6 +106,18 @@ def test_alert_proposal_part_still_goes_to_its_own_field(): assert result.unhandled_parts == [] +def test_workflow_status_part_stays_out_of_the_answer(): + result = parse_sse_lines( + _multipart_lines( + {"type": "workflowStatus", "skills": ["dashboard_editor"], "status": "ended"}, + {"type": "text", "text": "Done."}, + ) + ) + assert result.unhandled_parts == [] + assert result.search_results == [] + assert render_answer_text(result) == "Done." + + def test_known_part_types_matches_the_documented_gen_ai_union(): gen_ai_message_part_union = { "text", @@ -118,6 +130,7 @@ def test_known_part_types_matches_the_documented_gen_ai_union(): "searchResults", "alertProposal", "clarifyingQuestions", + "workflowStatus", } assert gen_ai_message_part_union == _KNOWN_PART_TYPES