From d28c9f0c8719af98dd26125b70fd7ed7c1618931 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 12 Nov 2026 07:45:41 +0100 Subject: [PATCH 1/2] fix(server): keep finish_reason length when reasoning exhausts the budget A thinking model whose private reasoning used the whole max_tokens budget was rewritten to finish_reason "error" with code empty_model_output. The gateway serves that code as HTTP 500, which generic OpenAI clients retry, and every retry fails the same way. Only a stop terminal without visible output is now stamped. A length terminal keeps its finish reason and returns an empty answer, which is the OpenAI contract for reasoning models and already tells the client that the budget ran out. The SDK READMEs and the gateway code docs describe the split. Refs #926 --- packages/sie_gateway/src/http_error.rs | 9 +-- packages/sie_sdk/README.md | 14 +++-- .../sie_server/adapters/_generation_base.py | 18 +++--- .../tests/adapters/test_generation_base.py | 63 ++++++++++++++++--- .../adapters/test_sglang_muse_channels.py | 5 +- .../tests/processors/test_streaming.py | 24 ++++++- packages/sie_ts_sdk/README.md | 5 +- 7 files changed, 105 insertions(+), 33 deletions(-) diff --git a/packages/sie_gateway/src/http_error.rs b/packages/sie_gateway/src/http_error.rs index a38bbffcc..c3deece71 100644 --- a/packages/sie_gateway/src/http_error.rs +++ b/packages/sie_gateway/src/http_error.rs @@ -144,10 +144,11 @@ pub mod openai_code { pub const RATE_LIMIT_EXCEEDED: &str = "rate_limit_exceeded"; /// OpenAI-compatible scale-from-zero / worker provisioning signal. pub const PROVISIONING: &str = "provisioning"; - /// Worker terminal for a generation that finished nominally but produced - /// no visible output text or tool calls (e.g. private reasoning consumed - /// the whole budget, #3104/#3136). Terminal and non-retryable: tokens were - /// genuinely consumed, so it settles exactly like ``inference_error``. + /// Worker terminal for a generation that stopped but produced no visible + /// output text or tool calls (e.g. only private reasoning, #3104/#3136). + /// A budget that runs out first keeps ``finish_reason: "length"`` instead. + /// Terminal and non-retryable: tokens were genuinely consumed, so it + /// settles exactly like ``inference_error``. pub const EMPTY_MODEL_OUTPUT: &str = "empty_model_output"; /// Worker terminal for a guard model whose first output positions carry /// no complete Yes/No verdict distribution. Terminal and non-retryable; diff --git a/packages/sie_sdk/README.md b/packages/sie_sdk/README.md index 648cd0429..4e52346f4 100644 --- a/packages/sie_sdk/README.md +++ b/packages/sie_sdk/README.md @@ -247,8 +247,9 @@ different dimension does not change that setting. Its configured threshold produces `Yes` (unsafe) or `No` (safe). A missing or invalid verdict returns `invalid_guard_verdict`; never treat an error or an empty response as safe. Private reasoning is hidden on both input surfaces. If it consumes the entire -generation budget without usable output, the request fails with -`empty_model_output`. +generation budget, the answer is empty with `finish_reason` `"length"`; raise +`max_tokens` to leave room for the answer. A model that stops without usable +output fails with `empty_model_output`. ## Connecting to SIE Serverless or a managed gateway @@ -466,10 +467,11 @@ the dropped items when every submitted item carried an `id`. Two generation-specific errors are terminal and never retried: -- `empty_model_output` — the generation finished nominally but produced - no visible output text (for example, private reasoning consumed the - whole token budget). Tokens were genuinely consumed, so the request - is not re-run. Surfaces as `ServerError` with +- `empty_model_output` — the generation stopped but produced no visible + output text (for example, the model emitted only private reasoning). + Tokens were genuinely consumed, so the request is not re-run. A + budget that runs out during reasoning is not this error: the answer + is empty with `finish_reason` `"length"`. Surfaces as `ServerError` with `code == "empty_model_output"`; on streaming calls it is raised mid-stream with the gateway request id attached for correlation. - `MODEL_OUTPUT_PARSE_ERROR` — a strict grammar (`grammar.strict` or diff --git a/packages/sie_server/src/sie_server/adapters/_generation_base.py b/packages/sie_server/src/sie_server/adapters/_generation_base.py index 4a750c51b..177f860cf 100644 --- a/packages/sie_server/src/sie_server/adapters/_generation_base.py +++ b/packages/sie_server/src/sie_server/adapters/_generation_base.py @@ -773,13 +773,15 @@ async def suppress_thinking_blocks( Closing the wrapper eagerly closes the upstream adapter iterator so client cancellation still reaches SGLang's ``/abort_request`` cleanup. - A stream that terminates without ever producing meaningful visible output - (no non-whitespace text on any choice or candidate, no tool calls) is - stamped with ``error_code="empty_model_output"`` on the terminal chunk — - e.g. when private reasoning consumed the entire generation budget. Without - the stamp such a request settles as a nominally successful but unusable + A stream that stops without ever producing meaningful visible output (no + non-whitespace text on any choice or candidate, no tool calls) is stamped + with ``error_code="empty_model_output"`` on the terminal chunk. Without the + stamp such a request settles as a nominally successful but unusable response that clients cannot distinguish from a real answer (#3104/#3136). - Cancelled and already-errored terminals are left untouched. + A ``length`` terminal keeps its finish reason even when private reasoning + consumed the whole budget: ``length`` already reports the truncation, as + the OpenAI contract does for reasoning models. Cancelled and + already-errored terminals are left untouched. """ states: dict[int, ThinkingBlockStripper] = {} emitted_visible_text = False @@ -856,10 +858,10 @@ async def suppress_thinking_blocks( and not emitted_meaningful_text and not emitted_tool_call and rewritten.error_code is None - and rewritten.finish_reason in ("stop", "length") + and rewritten.finish_reason == "stop" and not _candidates_have_visible_text(rewritten.candidates) ): - reason = " (private reasoning consumed the generation budget)" if hid_reasoning else "" + reason = " (the model produced only private reasoning)" if hid_reasoning else "" rewritten = replace( rewritten, finish_reason="error", diff --git a/packages/sie_server/tests/adapters/test_generation_base.py b/packages/sie_server/tests/adapters/test_generation_base.py index a2d755695..26053d285 100644 --- a/packages/sie_server/tests/adapters/test_generation_base.py +++ b/packages/sie_server/tests/adapters/test_generation_base.py @@ -292,8 +292,8 @@ async def test_suppress_thinking_blocks_hides_truncated_reasoning_and_its_logpro assert normalized[0].is_first is False assert normalized[0].logprobs is None assert normalized[1].done is True - assert normalized[1].finish_reason == "error" - assert normalized[1].error_code == "empty_model_output" + assert normalized[1].finish_reason == "length" + assert normalized[1].error_code is None assert normalized[1].completion_tokens == 2 assert normalized[1].text_delta == "" assert normalized[1].logprobs is None @@ -474,9 +474,9 @@ async def test_suppress_thinking_blocks_surfaces_standalone_close_failure() -> N @pytest.mark.asyncio -async def test_reasoning_consuming_the_whole_budget_stamps_empty_output_error() -> None: - """#3104/#3136: budget exhausted inside a think block -> only whitespace is - visible and the terminal chunk must carry the typed empty-output error. +async def test_reasoning_consuming_the_whole_budget_keeps_the_length_finish() -> None: + """Budget exhausted inside a think block: ``length`` already reports the + truncation, so the terminal keeps it instead of an empty-output error. """ source = _chunks( GenerationChunk(text_delta="\n\n", is_first=True), @@ -487,6 +487,28 @@ async def test_reasoning_consuming_the_whole_budget_stamps_empty_output_error() normalized = [chunk async for chunk in suppress_thinking_blocks(source)] + assert "".join(chunk.text_delta for chunk in normalized) == "\n\n" + terminal = normalized[-1] + assert terminal.done is True + assert terminal.finish_reason == "length" + assert terminal.error_code is None + assert terminal.error_message is None + assert terminal.completion_tokens == 32 + + +@pytest.mark.asyncio +async def test_reasoning_only_answer_that_stops_stamps_empty_output_error() -> None: + """#3104/#3136: a model that stops after only private reasoning must not + settle as a nominally successful empty answer. + """ + source = _chunks( + GenerationChunk(text_delta="\n\n", is_first=True), + GenerationChunk(text_delta="private reasoning"), + GenerationChunk(text_delta="", done=True, finish_reason="stop", completion_tokens=32), + ) + + normalized = [chunk async for chunk in suppress_thinking_blocks(source)] + assert "".join(chunk.text_delta for chunk in normalized) == "\n\n" terminal = normalized[-1] assert terminal.done is True @@ -573,14 +595,14 @@ async def test_candidates_with_visible_text_are_not_stamped() -> None: @pytest.mark.asyncio async def test_candidates_all_reasoning_are_stamped() -> None: - """Non-streaming n>1 where every candidate was reasoning-only.""" + """Non-streaming n>1 where every candidate stopped after reasoning only.""" terminal = GenerationChunk( text_delta="", done=True, - finish_reason="length", + finish_reason="stop", candidates=( - {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, - {"index": 1, "text": "more truncated reasoning", "finish_reason": "length"}, + {"index": 0, "text": "reasoning", "finish_reason": "stop"}, + {"index": 1, "text": "more reasoning", "finish_reason": "stop"}, ), ) @@ -591,6 +613,26 @@ async def test_candidates_all_reasoning_are_stamped() -> None: assert "reasoning" in normalized.error_message +@pytest.mark.asyncio +async def test_truncated_reasoning_candidates_keep_the_length_finish() -> None: + """Non-streaming n>1 where every candidate ran out of budget while reasoning.""" + terminal = GenerationChunk( + text_delta="", + done=True, + finish_reason="length", + candidates=( + {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, + {"index": 1, "text": "more truncated reasoning", "finish_reason": "length"}, + ), + ) + + [normalized] = [chunk async for chunk in suppress_thinking_blocks(_chunks(terminal))] + + assert normalized.finish_reason == "length" + assert normalized.error_code is None + assert [candidate["text"] for candidate in normalized.candidates or ()] == ["", ""] + + @pytest.mark.asyncio async def test_collect_generation_carries_terminal_error_fields() -> None: """#3104/#3136: buffered aggregation must not drop the terminal chunk's @@ -710,7 +752,8 @@ async def test_suppress_thinking_blocks_keeps_truncated_reasoning_when_asked() - assert "".join(chunk.reasoning_delta for chunk in kept) == "cut off mid" assert "".join(chunk.text_delta for chunk in kept) == "" - assert kept[-1].error_code == "empty_model_output" + assert kept[-1].finish_reason == "length" + assert kept[-1].error_code is None @pytest.mark.asyncio diff --git a/packages/sie_server/tests/adapters/test_sglang_muse_channels.py b/packages/sie_server/tests/adapters/test_sglang_muse_channels.py index 3c78179b5..7f9db8770 100644 --- a/packages/sie_server/tests/adapters/test_sglang_muse_channels.py +++ b/packages/sie_server/tests/adapters/test_sglang_muse_channels.py @@ -314,7 +314,7 @@ def test_worker_pipeline_converts_arguments_by_their_declared_type() -> None: assert json.loads(deltas[1].arguments_delta) == {"days": 3} -def test_reasoning_that_uses_the_whole_budget_is_an_explicit_error() -> None: +def test_reasoning_that_uses_the_whole_budget_keeps_the_length_finish() -> None: stream = suppress_thinking_blocks( translate_muse_channels(_chunks(" to=self<|message|>Still", finish_reason="length")) ) @@ -322,7 +322,8 @@ def test_reasoning_that_uses_the_whole_budget_is_an_explicit_error() -> None: chunks = asyncio.run(_collect(stream)) assert _text(chunks) == "" - assert chunks[-1].error_code == "empty_model_output" + assert chunks[-1].finish_reason == "length" + assert chunks[-1].error_code is None def test_adapter_translates_what_the_base_adapter_streams() -> None: diff --git a/packages/sie_server/tests/processors/test_streaming.py b/packages/sie_server/tests/processors/test_streaming.py index d65acda55..dd2f7bb76 100644 --- a/packages/sie_server/tests/processors/test_streaming.py +++ b/packages/sie_server/tests/processors/test_streaming.py @@ -5008,6 +5008,27 @@ async def test_queued_chat_include_reasoning_publishes_text_and_reasoning_in_mod @pytest.mark.asyncio async def test_queued_chat_include_reasoning_still_fails_an_answer_that_is_only_reasoning( monkeypatch: pytest.MonkeyPatch, +) -> None: + nc, proc = _reasoning_processor( + monkeypatch, + [ + GenerationChunk(text_delta="thinking and then stopping", is_first=True), + GenerationChunk(text_delta="", done=True, finish_reason="stop", prompt_tokens=1, completion_tokens=8), + ], + ) + + await proc.process(_make_msg(_reasoning_work(stream=True)), "Qwen/Qwen3.5-4B") + + decoded = _decode_chunks(nc) + assert "".join(chunk.get("reasoning_delta", "") for chunk in decoded) == "thinking and then stopping" + assert not any(chunk.get("text_delta") for chunk in decoded) + assert decoded[-1]["done"] is True + assert decoded[-1]["error"]["code"] == "empty_model_output" + + +@pytest.mark.asyncio +async def test_queued_chat_include_reasoning_reports_truncated_reasoning_as_length( + monkeypatch: pytest.MonkeyPatch, ) -> None: nc, proc = _reasoning_processor( monkeypatch, @@ -5023,4 +5044,5 @@ async def test_queued_chat_include_reasoning_still_fails_an_answer_that_is_only_ assert "".join(chunk.get("reasoning_delta", "") for chunk in decoded) == "thinking until the budget ends" assert not any(chunk.get("text_delta") for chunk in decoded) assert decoded[-1]["done"] is True - assert decoded[-1]["error"]["code"] == "empty_model_output" + assert decoded[-1]["finish_reason"] == "length" + assert "error" not in decoded[-1] diff --git a/packages/sie_ts_sdk/README.md b/packages/sie_ts_sdk/README.md index 346ab773b..a85d1b12f 100644 --- a/packages/sie_ts_sdk/README.md +++ b/packages/sie_ts_sdk/README.md @@ -160,8 +160,9 @@ Granite Guardian's shipped risk dimension is `harm`; requesting another dimension in prompt text does not change it. Valid thresholded verdicts are `Yes` (unsafe) and `No` (safe); missing or invalid verdicts fail with `invalid_guard_verdict`. Never interpret an error or empty output as safe. -Reasoning remains private; a budget exhausted without usable output fails with -`empty_model_output`. +Reasoning remains private. A budget exhausted during reasoning returns an empty +answer whose finish reason is `"length"`; a model that stops without usable output +fails with `empty_model_output`. ```typescript // Aggregated result From 4ff320d5c0a150898d8a60f991c78aca6c7f892a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 12 Nov 2026 09:13:09 +0100 Subject: [PATCH 2/2] fix(server): leave an n>1 answer unstamped when a candidate hit length --- .../sie_server/adapters/_generation_base.py | 6 +++++ .../tests/adapters/test_generation_base.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/packages/sie_server/src/sie_server/adapters/_generation_base.py b/packages/sie_server/src/sie_server/adapters/_generation_base.py index 177f860cf..ae19a25da 100644 --- a/packages/sie_server/src/sie_server/adapters/_generation_base.py +++ b/packages/sie_server/src/sie_server/adapters/_generation_base.py @@ -704,6 +704,11 @@ def _candidates_have_visible_text(candidates: tuple[dict[str, Any], ...] | None) return any(isinstance(text := candidate.get("text"), str) and text.strip() for candidate in candidates) +def _candidates_hit_length(candidates: tuple[dict[str, Any], ...] | None) -> bool: + """Return whether any ``n > 1`` candidate aggregate ran out of budget.""" + return any(candidate.get("finish_reason") == "length" for candidate in candidates or ()) + + async def aclose_with_error_precedence( resource: Any, *, @@ -859,6 +864,7 @@ async def suppress_thinking_blocks( and not emitted_tool_call and rewritten.error_code is None and rewritten.finish_reason == "stop" + and not _candidates_hit_length(rewritten.candidates) and not _candidates_have_visible_text(rewritten.candidates) ): reason = " (the model produced only private reasoning)" if hid_reasoning else "" diff --git a/packages/sie_server/tests/adapters/test_generation_base.py b/packages/sie_server/tests/adapters/test_generation_base.py index 26053d285..204ff1f89 100644 --- a/packages/sie_server/tests/adapters/test_generation_base.py +++ b/packages/sie_server/tests/adapters/test_generation_base.py @@ -633,6 +633,29 @@ async def test_truncated_reasoning_candidates_keep_the_length_finish() -> None: assert [candidate["text"] for candidate in normalized.candidates or ()] == ["", ""] +@pytest.mark.asyncio +async def test_a_truncated_candidate_keeps_a_stop_terminal_unstamped() -> None: + """SGLang's n>1 terminal reports ``stop`` at the top level; a candidate that + ran out of budget during reasoning keeps its own ``length`` instead of + turning the whole answer into an empty-output error. + """ + terminal = GenerationChunk( + text_delta="", + done=True, + finish_reason="stop", + candidates=( + {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, + {"index": 1, "text": "reasoning", "finish_reason": "stop"}, + ), + ) + + [normalized] = [chunk async for chunk in suppress_thinking_blocks(_chunks(terminal))] + + assert normalized.finish_reason == "stop" + assert normalized.error_code is None + assert [candidate["finish_reason"] for candidate in normalized.candidates or ()] == ["length", "stop"] + + @pytest.mark.asyncio async def test_collect_generation_carries_terminal_error_fields() -> None: """#3104/#3136: buffered aggregation must not drop the terminal chunk's