diff --git a/packages/sie_gateway/src/http_error.rs b/packages/sie_gateway/src/http_error.rs index a38bbffcc..c3deece71 100644 --- a/packages/sie_gateway/src/http_error.rs +++ b/packages/sie_gateway/src/http_error.rs @@ -144,10 +144,11 @@ pub mod openai_code { pub const RATE_LIMIT_EXCEEDED: &str = "rate_limit_exceeded"; /// OpenAI-compatible scale-from-zero / worker provisioning signal. pub const PROVISIONING: &str = "provisioning"; - /// Worker terminal for a generation that finished nominally but produced - /// no visible output text or tool calls (e.g. private reasoning consumed - /// the whole budget, #3104/#3136). Terminal and non-retryable: tokens were - /// genuinely consumed, so it settles exactly like ``inference_error``. + /// Worker terminal for a generation that stopped but produced no visible + /// output text or tool calls (e.g. only private reasoning, #3104/#3136). + /// A budget that runs out first keeps ``finish_reason: "length"`` instead. + /// Terminal and non-retryable: tokens were genuinely consumed, so it + /// settles exactly like ``inference_error``. pub const EMPTY_MODEL_OUTPUT: &str = "empty_model_output"; /// Worker terminal for a guard model whose first output positions carry /// no complete Yes/No verdict distribution. Terminal and non-retryable; diff --git a/packages/sie_sdk/README.md b/packages/sie_sdk/README.md index 648cd0429..4e52346f4 100644 --- a/packages/sie_sdk/README.md +++ b/packages/sie_sdk/README.md @@ -247,8 +247,9 @@ different dimension does not change that setting. Its configured threshold produces `Yes` (unsafe) or `No` (safe). A missing or invalid verdict returns `invalid_guard_verdict`; never treat an error or an empty response as safe. Private reasoning is hidden on both input surfaces. If it consumes the entire -generation budget without usable output, the request fails with -`empty_model_output`. +generation budget, the answer is empty with `finish_reason` `"length"`; raise +`max_tokens` to leave room for the answer. A model that stops without usable +output fails with `empty_model_output`. ## Connecting to SIE Serverless or a managed gateway @@ -466,10 +467,11 @@ the dropped items when every submitted item carried an `id`. Two generation-specific errors are terminal and never retried: -- `empty_model_output` — the generation finished nominally but produced - no visible output text (for example, private reasoning consumed the - whole token budget). Tokens were genuinely consumed, so the request - is not re-run. Surfaces as `ServerError` with +- `empty_model_output` — the generation stopped but produced no visible + output text (for example, the model emitted only private reasoning). + Tokens were genuinely consumed, so the request is not re-run. A + budget that runs out during reasoning is not this error: the answer + is empty with `finish_reason` `"length"`. Surfaces as `ServerError` with `code == "empty_model_output"`; on streaming calls it is raised mid-stream with the gateway request id attached for correlation. - `MODEL_OUTPUT_PARSE_ERROR` — a strict grammar (`grammar.strict` or diff --git a/packages/sie_server/src/sie_server/adapters/_generation_base.py b/packages/sie_server/src/sie_server/adapters/_generation_base.py index 4a750c51b..ae19a25da 100644 --- a/packages/sie_server/src/sie_server/adapters/_generation_base.py +++ b/packages/sie_server/src/sie_server/adapters/_generation_base.py @@ -704,6 +704,11 @@ def _candidates_have_visible_text(candidates: tuple[dict[str, Any], ...] | None) return any(isinstance(text := candidate.get("text"), str) and text.strip() for candidate in candidates) +def _candidates_hit_length(candidates: tuple[dict[str, Any], ...] | None) -> bool: + """Return whether any ``n > 1`` candidate aggregate ran out of budget.""" + return any(candidate.get("finish_reason") == "length" for candidate in candidates or ()) + + async def aclose_with_error_precedence( resource: Any, *, @@ -773,13 +778,15 @@ async def suppress_thinking_blocks( Closing the wrapper eagerly closes the upstream adapter iterator so client cancellation still reaches SGLang's ``/abort_request`` cleanup. - A stream that terminates without ever producing meaningful visible output - (no non-whitespace text on any choice or candidate, no tool calls) is - stamped with ``error_code="empty_model_output"`` on the terminal chunk — - e.g. when private reasoning consumed the entire generation budget. Without - the stamp such a request settles as a nominally successful but unusable + A stream that stops without ever producing meaningful visible output (no + non-whitespace text on any choice or candidate, no tool calls) is stamped + with ``error_code="empty_model_output"`` on the terminal chunk. Without the + stamp such a request settles as a nominally successful but unusable response that clients cannot distinguish from a real answer (#3104/#3136). - Cancelled and already-errored terminals are left untouched. + A ``length`` terminal keeps its finish reason even when private reasoning + consumed the whole budget: ``length`` already reports the truncation, as + the OpenAI contract does for reasoning models. Cancelled and + already-errored terminals are left untouched. """ states: dict[int, ThinkingBlockStripper] = {} emitted_visible_text = False @@ -856,10 +863,11 @@ async def suppress_thinking_blocks( and not emitted_meaningful_text and not emitted_tool_call and rewritten.error_code is None - and rewritten.finish_reason in ("stop", "length") + and rewritten.finish_reason == "stop" + and not _candidates_hit_length(rewritten.candidates) and not _candidates_have_visible_text(rewritten.candidates) ): - reason = " (private reasoning consumed the generation budget)" if hid_reasoning else "" + reason = " (the model produced only private reasoning)" if hid_reasoning else "" rewritten = replace( rewritten, finish_reason="error", diff --git a/packages/sie_server/tests/adapters/test_generation_base.py b/packages/sie_server/tests/adapters/test_generation_base.py index a2d755695..204ff1f89 100644 --- a/packages/sie_server/tests/adapters/test_generation_base.py +++ b/packages/sie_server/tests/adapters/test_generation_base.py @@ -292,8 +292,8 @@ async def test_suppress_thinking_blocks_hides_truncated_reasoning_and_its_logpro assert normalized[0].is_first is False assert normalized[0].logprobs is None assert normalized[1].done is True - assert normalized[1].finish_reason == "error" - assert normalized[1].error_code == "empty_model_output" + assert normalized[1].finish_reason == "length" + assert normalized[1].error_code is None assert normalized[1].completion_tokens == 2 assert normalized[1].text_delta == "" assert normalized[1].logprobs is None @@ -474,9 +474,9 @@ async def test_suppress_thinking_blocks_surfaces_standalone_close_failure() -> N @pytest.mark.asyncio -async def test_reasoning_consuming_the_whole_budget_stamps_empty_output_error() -> None: - """#3104/#3136: budget exhausted inside a think block -> only whitespace is - visible and the terminal chunk must carry the typed empty-output error. +async def test_reasoning_consuming_the_whole_budget_keeps_the_length_finish() -> None: + """Budget exhausted inside a think block: ``length`` already reports the + truncation, so the terminal keeps it instead of an empty-output error. """ source = _chunks( GenerationChunk(text_delta="\n\n", is_first=True), @@ -487,6 +487,28 @@ async def test_reasoning_consuming_the_whole_budget_stamps_empty_output_error() normalized = [chunk async for chunk in suppress_thinking_blocks(source)] + assert "".join(chunk.text_delta for chunk in normalized) == "\n\n" + terminal = normalized[-1] + assert terminal.done is True + assert terminal.finish_reason == "length" + assert terminal.error_code is None + assert terminal.error_message is None + assert terminal.completion_tokens == 32 + + +@pytest.mark.asyncio +async def test_reasoning_only_answer_that_stops_stamps_empty_output_error() -> None: + """#3104/#3136: a model that stops after only private reasoning must not + settle as a nominally successful empty answer. + """ + source = _chunks( + GenerationChunk(text_delta="\n\n", is_first=True), + GenerationChunk(text_delta="private reasoning"), + GenerationChunk(text_delta="", done=True, finish_reason="stop", completion_tokens=32), + ) + + normalized = [chunk async for chunk in suppress_thinking_blocks(source)] + assert "".join(chunk.text_delta for chunk in normalized) == "\n\n" terminal = normalized[-1] assert terminal.done is True @@ -573,14 +595,14 @@ async def test_candidates_with_visible_text_are_not_stamped() -> None: @pytest.mark.asyncio async def test_candidates_all_reasoning_are_stamped() -> None: - """Non-streaming n>1 where every candidate was reasoning-only.""" + """Non-streaming n>1 where every candidate stopped after reasoning only.""" terminal = GenerationChunk( text_delta="", done=True, - finish_reason="length", + finish_reason="stop", candidates=( - {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, - {"index": 1, "text": "more truncated reasoning", "finish_reason": "length"}, + {"index": 0, "text": "reasoning", "finish_reason": "stop"}, + {"index": 1, "text": "more reasoning", "finish_reason": "stop"}, ), ) @@ -591,6 +613,49 @@ async def test_candidates_all_reasoning_are_stamped() -> None: assert "reasoning" in normalized.error_message +@pytest.mark.asyncio +async def test_truncated_reasoning_candidates_keep_the_length_finish() -> None: + """Non-streaming n>1 where every candidate ran out of budget while reasoning.""" + terminal = GenerationChunk( + text_delta="", + done=True, + finish_reason="length", + candidates=( + {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, + {"index": 1, "text": "more truncated reasoning", "finish_reason": "length"}, + ), + ) + + [normalized] = [chunk async for chunk in suppress_thinking_blocks(_chunks(terminal))] + + assert normalized.finish_reason == "length" + assert normalized.error_code is None + assert [candidate["text"] for candidate in normalized.candidates or ()] == ["", ""] + + +@pytest.mark.asyncio +async def test_a_truncated_candidate_keeps_a_stop_terminal_unstamped() -> None: + """SGLang's n>1 terminal reports ``stop`` at the top level; a candidate that + ran out of budget during reasoning keeps its own ``length`` instead of + turning the whole answer into an empty-output error. + """ + terminal = GenerationChunk( + text_delta="", + done=True, + finish_reason="stop", + candidates=( + {"index": 0, "text": "truncated reasoning", "finish_reason": "length"}, + {"index": 1, "text": "reasoning", "finish_reason": "stop"}, + ), + ) + + [normalized] = [chunk async for chunk in suppress_thinking_blocks(_chunks(terminal))] + + assert normalized.finish_reason == "stop" + assert normalized.error_code is None + assert [candidate["finish_reason"] for candidate in normalized.candidates or ()] == ["length", "stop"] + + @pytest.mark.asyncio async def test_collect_generation_carries_terminal_error_fields() -> None: """#3104/#3136: buffered aggregation must not drop the terminal chunk's @@ -710,7 +775,8 @@ async def test_suppress_thinking_blocks_keeps_truncated_reasoning_when_asked() - assert "".join(chunk.reasoning_delta for chunk in kept) == "cut off mid" assert "".join(chunk.text_delta for chunk in kept) == "" - assert kept[-1].error_code == "empty_model_output" + assert kept[-1].finish_reason == "length" + assert kept[-1].error_code is None @pytest.mark.asyncio diff --git a/packages/sie_server/tests/adapters/test_sglang_muse_channels.py b/packages/sie_server/tests/adapters/test_sglang_muse_channels.py index 3c78179b5..7f9db8770 100644 --- a/packages/sie_server/tests/adapters/test_sglang_muse_channels.py +++ b/packages/sie_server/tests/adapters/test_sglang_muse_channels.py @@ -314,7 +314,7 @@ def test_worker_pipeline_converts_arguments_by_their_declared_type() -> None: assert json.loads(deltas[1].arguments_delta) == {"days": 3} -def test_reasoning_that_uses_the_whole_budget_is_an_explicit_error() -> None: +def test_reasoning_that_uses_the_whole_budget_keeps_the_length_finish() -> None: stream = suppress_thinking_blocks( translate_muse_channels(_chunks(" to=self<|message|>Still", finish_reason="length")) ) @@ -322,7 +322,8 @@ def test_reasoning_that_uses_the_whole_budget_is_an_explicit_error() -> None: chunks = asyncio.run(_collect(stream)) assert _text(chunks) == "" - assert chunks[-1].error_code == "empty_model_output" + assert chunks[-1].finish_reason == "length" + assert chunks[-1].error_code is None def test_adapter_translates_what_the_base_adapter_streams() -> None: diff --git a/packages/sie_server/tests/processors/test_streaming.py b/packages/sie_server/tests/processors/test_streaming.py index d65acda55..dd2f7bb76 100644 --- a/packages/sie_server/tests/processors/test_streaming.py +++ b/packages/sie_server/tests/processors/test_streaming.py @@ -5008,6 +5008,27 @@ async def test_queued_chat_include_reasoning_publishes_text_and_reasoning_in_mod @pytest.mark.asyncio async def test_queued_chat_include_reasoning_still_fails_an_answer_that_is_only_reasoning( monkeypatch: pytest.MonkeyPatch, +) -> None: + nc, proc = _reasoning_processor( + monkeypatch, + [ + GenerationChunk(text_delta="thinking and then stopping", is_first=True), + GenerationChunk(text_delta="", done=True, finish_reason="stop", prompt_tokens=1, completion_tokens=8), + ], + ) + + await proc.process(_make_msg(_reasoning_work(stream=True)), "Qwen/Qwen3.5-4B") + + decoded = _decode_chunks(nc) + assert "".join(chunk.get("reasoning_delta", "") for chunk in decoded) == "thinking and then stopping" + assert not any(chunk.get("text_delta") for chunk in decoded) + assert decoded[-1]["done"] is True + assert decoded[-1]["error"]["code"] == "empty_model_output" + + +@pytest.mark.asyncio +async def test_queued_chat_include_reasoning_reports_truncated_reasoning_as_length( + monkeypatch: pytest.MonkeyPatch, ) -> None: nc, proc = _reasoning_processor( monkeypatch, @@ -5023,4 +5044,5 @@ async def test_queued_chat_include_reasoning_still_fails_an_answer_that_is_only_ assert "".join(chunk.get("reasoning_delta", "") for chunk in decoded) == "thinking until the budget ends" assert not any(chunk.get("text_delta") for chunk in decoded) assert decoded[-1]["done"] is True - assert decoded[-1]["error"]["code"] == "empty_model_output" + assert decoded[-1]["finish_reason"] == "length" + assert "error" not in decoded[-1] diff --git a/packages/sie_ts_sdk/README.md b/packages/sie_ts_sdk/README.md index 346ab773b..a85d1b12f 100644 --- a/packages/sie_ts_sdk/README.md +++ b/packages/sie_ts_sdk/README.md @@ -160,8 +160,9 @@ Granite Guardian's shipped risk dimension is `harm`; requesting another dimension in prompt text does not change it. Valid thresholded verdicts are `Yes` (unsafe) and `No` (safe); missing or invalid verdicts fail with `invalid_guard_verdict`. Never interpret an error or empty output as safe. -Reasoning remains private; a budget exhausted without usable output fails with -`empty_model_output`. +Reasoning remains private. A budget exhausted during reasoning returns an empty +answer whose finish reason is `"length"`; a model that stops without usable output +fails with `empty_model_output`. ```typescript // Aggregated result