From 0f71836aaa01459b1c8375261c0c6b51cb7bfb9d Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 00:33:25 -0400 Subject: [PATCH 01/18] fix(login): accept real API keys and ask for --yes instead of cancelling dg login no longer checks for an sk-/pk- prefix: Deepgram keys don't have one, and login_with_api_key already verifies the key with the API. A 40-hex key used to stop with "Login cancelled" whenever no one could answer the "Continue anyway?" prompt. The two login confirmations (DEEPGRAM_API_KEY is set; the profile already has a saved login) are asked only of a person at a terminal. In any other run they return confirmation_required, exit 1, with details.flag "--yes" and a next_step that reruns the command with --yes, or null when the command line holds a key. They no longer return "cancelled", which is kept for a person saying no. -y/--yes answers both; -f/--force-write stays as the same flag. BaseCommand gains can_prompt(), the check confirm() already made, so a command can tell "no one can answer" from "the answer was no". --- .../src/deepctl_cmd_login/command.py | 181 ++++++++++- .../src/deepctl_cmd_login/models.py | 22 ++ .../tests/unit/test_login_command.py | 6 +- .../tests/unit/test_login_confirm.py | 307 ++++++++++++++++++ .../deepctl-core/src/deepctl_core/auth.py | 6 - .../src/deepctl_core/base_command.py | 17 +- .../deepctl-core/src/deepctl_core/messages.py | 28 ++ src/deepctl/_manifest.d/login.json | 24 +- 8 files changed, 564 insertions(+), 27 deletions(-) create mode 100644 packages/deepctl-cmd-login/tests/unit/test_login_confirm.py diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index bf6d937f..820c12ff 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1,7 +1,11 @@ """Login command for deepctl.""" +import os +import shlex +import shutil import sys from typing import Any +from urllib.parse import urlsplit import click from deepctl_core import ( @@ -45,16 +49,129 @@ from rich.markup import escape from rich.prompt import Prompt -from .models import LoginResult, LogoutResult, WhoamiResult +from .models import LoginCodedError, LoginResult, LogoutResult, WhoamiResult console = Console() +_PROG_NAMES = ("dg", "deepgram", "deepctl") +_YES_FLAG = "--yes" +# Options whose value is a key. +_SECRET_OPTIONS = ("--api-key", "-k") + def _interactive_terminal() -> bool: """A person can answer a prompt: stdin and stderr are both terminals.""" return sys.stdin.isatty() and sys.stderr.isatty() +def _prog() -> str | None: + """``{prog}``: the running executable, as a command. + + The bare name when ``which(name)`` is this same file; else the console + script's absolute path, quoted. ``None`` when neither applies (``python + -m``, an unknown launcher), which makes ``next_step`` null. + """ + argv0 = sys.argv[0] if sys.argv else "" + name = os.path.basename(argv0) + if os.name == "nt" and name.lower().endswith(".exe"): + name = name[:-4] + if name not in _PROG_NAMES: + return None + found = shutil.which(name) + running = shutil.which(argv0) or argv0 + try: + if found and os.path.samefile(found, running): + return name + except OSError: + pass + if not os.path.isfile(running): + return None + path = os.path.abspath(running) + if os.name == "nt": + return None if shlex.quote(path) != path else path + return shlex.quote(path) + + +def _secret_in(argv: list[str]) -> bool: + """True when ``argv`` holds a key or a URL with userinfo or a query.""" + after_option = False + for token in argv: + if after_option: + return True + if token in _SECRET_OPTIONS: + after_option = True + continue + if token.startswith("--api-key=") or ( + token.startswith("-k") and not token.startswith("--") and len(token) > 2 + ): + return True + if "://" in token: + try: + parts = urlsplit(token) + except ValueError: + return True + if "@" in parts.netloc or parts.query: + return True + return False + + +def _rerun_with_yes(prog: str | None) -> str | None: + """``{prog} {argv} --yes`` as a runnable command, or None.""" + argv = sys.argv[1:] + if prog is None or _secret_in(argv): + return None + tokens = [*argv, _YES_FLAG] + if any(token.startswith("=") for token in tokens): + return None # zsh's EQUALS option would expand it + if os.name == "nt" and any(shlex.quote(token) != token for token in tokens): + return None + step = f"{prog} {shlex.join(tokens)}" + return step if shlex.join(shlex.split(step)) == step else None + + +def _hint_for( + human: str, + agent: str, + next_step: str | None, + *, + agent_null: str | None = None, + automation_null: str | None = None, +) -> str: + """The hint for this run's mode, null-aware.""" + if is_ai_agent(): + return agent_null if next_step is None and agent_null else agent + if _output_config["agentic"]: + if next_step is None: + return automation_null or agent_null or agent + return agent + return human + + +def _coded_error( + error: str, + code: str, + hint: str, + next_step: str | None, + details: dict[str, Any] | None = None, +) -> LoginCodedError: + """Report a coded error on stderr and return its payload.""" + print_error(escape(error)) + print_hint(escape(hint)) + return LoginCodedError( + error=error, code=code, hint=hint, next_step=next_step, details=details + ) + + +def _confirmation_required(error: str, next_step: str | None, hint: str) -> Any: + return _coded_error( + error, + msg.CODE_CONFIRMATION_REQUIRED, + hint, + next_step, + {"flag": _YES_FLAG}, + ) + + class LoginCommand(BaseCommand): """Login command for authenticating with Deepgram.""" @@ -102,9 +219,18 @@ def get_arguments(self) -> list[dict[str, Any]]: "required": False, "is_option": True, }, + { + "names": ["--yes", "-y"], + "help": ( + "Save without asking, even if DEEPGRAM_API_KEY is set or the " + "profile already has a saved login (--save-base-url still asks)" + ), + "is_flag": True, + "is_option": True, + }, { "names": ["--force-write", "-f"], - "help": ("Don't prompt for confirmation when providing credentials"), + "help": "Same as --yes", "is_flag": True, "is_option": True, }, @@ -136,7 +262,7 @@ def handle( """Handle login command.""" api_key = kwargs.get("api_key") project_id = kwargs.get("project_id") - force_write = kwargs.get("force_write", False) + force_write = bool(kwargs.get("yes") or kwargs.get("force_write")) profile = kwargs.get("profile") save_base_url = kwargs.get("save_base_url", False) @@ -170,6 +296,23 @@ def handle( has_env_key, has_env_project = auth_manager.has_env_credentials() if has_env_key and not force_write: + if not self._can_ask(): + # Nobody can answer: say which flag does. + prog = _prog() + next_step = None if api_key else _rerun_with_yes(prog) + return _confirmation_required( + msg.E_CONFIRM_ENVKEY.format(prog=prog or _PROG_NAMES[0]), + next_step, + _hint_for( + msg.H_CONFIRM_YES, + msg.A_CONFIRM_ENVKEY, + next_step, + agent_null=msg.A_CONFIRM_ENVKEY_NULL.format( + prog=prog or _PROG_NAMES[0] + ), + automation_null=msg.C_CONFIRM_ENVKEY_NULL, + ), + ) console.print( "[yellow]Environment variable DEEPGRAM_API_KEY is set.[/yellow]" ) @@ -205,6 +348,19 @@ def handle( ) if has_profile_key and not force_write: + if not self._can_ask(): + next_step = _rerun_with_yes(_prog()) + return _confirmation_required( + msg.E_CONFIRM_RELOGIN.format(profile=msg.display(current_profile)), + next_step, + _hint_for( + msg.H_CONFIRM_YES, + msg.A_CONFIRM_YES, + next_step, + agent_null=msg.A_CONFIRM_YES_NULL, + automation_null=msg.C_CONFIRM_YES_NULL, + ), + ) console.print( f"[yellow]Already logged in to profile:[/yellow] {current_profile}" ) @@ -253,6 +409,10 @@ def handle( return result + def _can_ask(self) -> bool: + """A person can answer login's confirmations here.""" + return self.can_prompt() and _interactive_terminal() and not is_ai_agent() + def _confirm_save_base_url(self, config: Config, profile: str) -> Any | None: """Ask a person to approve saving the base URL. @@ -490,19 +650,8 @@ def _cli_auth( """Handle CLI authentication with API key.""" console.print("[blue]Configuring CLI with API key...[/blue]") - # Validate API key format - if not api_key.startswith(("sk-", "pk-")): - console.print( - "[yellow]Warning:[/yellow] API key format doesn't match " - "expected pattern" - ) - if not force_write and not self.confirm("Continue anyway?", default=False): - return LoginResult( - status="cancelled", - message="Login cancelled by user", - profile=config.profile or "default", - api_key_masked=None, - ) + # The key's format isn't checked: Deepgram keys have no fixed prefix, + # and login_with_api_key verifies the key with the API. # Validate project ID is provided with API key if not project_id: diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py index 1363f330..4355d336 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py @@ -1,9 +1,31 @@ """Models for login command.""" +from typing import Any + from deepctl_core import BaseResult +from deepctl_core.models import CodedErrorResult from pydantic import Field +class LoginCodedError(CodedErrorResult): + """A 0.5.0 coded error from ``dg login``, keys in the catalogue's order. + + ``next_step`` is always present (null when no command fixes it); + ``details`` is left out when there are none. The 0.4.1 coded + errors login also returns keep their keys. + """ + + next_step: str | None = None + schema_version: int = 1 + details: dict[str, Any] | None = None + + def model_dump(self, **kwargs: Any) -> dict[str, Any]: + data = super().model_dump(**kwargs) + if not data.get("details"): + data.pop("details", None) + return data + + class LoginResult(BaseResult): """Return structure for `deepctl login` command.""" diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_command.py b/packages/deepctl-cmd-login/tests/unit/test_login_command.py index 71019c7b..1a913932 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_command.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_command.py @@ -183,8 +183,10 @@ def mock_client(): @pytest.fixture def login_command(): - """Create a login command instance.""" - return LoginCommand() + """A login command run by a person in a terminal (its prompts can show).""" + cmd = LoginCommand() + cmd._can_ask = lambda: True # type: ignore[method-assign] + return cmd class TestLoginCommand: diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_confirm.py b/packages/deepctl-cmd-login/tests/unit/test_login_confirm.py new file mode 100644 index 00000000..4dc72000 --- /dev/null +++ b/packages/deepctl-cmd-login/tests/unit/test_login_confirm.py @@ -0,0 +1,307 @@ +"""`dg login` takes any key the API accepts, and a confirmation nobody can +answer is `confirmation_required` naming --yes, never a silent cancel.""" + +from __future__ import annotations + +import json +import sys + +import pytest +import yaml +from deepctl_cmd_login import command as login_module +from deepctl_cmd_login.command import LoginCommand +from deepctl_cmd_login.models import LoginCodedError +from deepctl_core import AuthManager, Config, DeepgramClient, hosts, output +from deepctl_core import auth as auth_module + +HEX_KEY = "0123456789abcdef0123456789abcdef01234567" # 40 hex, no prefix +SK_KEY = "sk-LGNFAKEKEY0000000000000000" + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setattr( + "deepctl_core.config._default_config_file", + lambda: tmp_path / "user" / "config.yaml", + ) + monkeypatch.setattr( + "deepctl_core.config._trust_root_location", + lambda: (tmp_path, tmp_path / "user" / "config.yaml"), + ) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(output, "_hints_shown", set()) + for name in ("is_agentic", "is_ai_agent"): + monkeypatch.setattr(output, name, lambda: False) + monkeypatch.setattr(login_module, name, lambda: False) + monkeypatch.setattr(login_module, "_interactive_terminal", lambda: False) + monkeypatch.setattr(login_module, "_prog", lambda: "dg") + monkeypatch.setattr(sys, "argv", ["dg", "login"]) + monkeypatch.delenv("DEEPGRAM_API_KEY", raising=False) + monkeypatch.delenv("DEEPGRAM_PROJECT_ID", raising=False) + saved = dict(output._output_config) + output._output_config.update(agentic=True, quiet=False, format="json") + monkeypatch.setattr(output.stderr_console, "_width", 400) + stored: dict[str, str] = {} + monkeypatch.setattr( + auth_module.keyring, + "set_password", + lambda service, name, value: stored.__setitem__(name, value), + ) + monkeypatch.setattr( + auth_module.keyring, "get_password", lambda service, name: stored.get(name) + ) + monkeypatch.setattr( + auth_module.keyring, "delete_password", lambda service, name: stored.pop(name) + ) + monkeypatch.setattr(LoginCommand, "_maybe_prompt_skills_setup", lambda self: None) + monkeypatch.setattr( + LoginCommand, "confirm", lambda *a, **k: pytest.fail("prompted") + ) + yield stored + output._output_config.clear() + output._output_config.update(saved) + + +@pytest.fixture +def keyring_store(_isolated): + return _isolated + + +@pytest.fixture +def verified(monkeypatch): + """The key check succeeds without a network call; records each key.""" + keys: list[str] = [] + + def verify(self, api_key=None, project_id=None): + keys.append(api_key) + return True, "ok", None + + monkeypatch.setattr(AuthManager, "verify_credentials", verify) + return keys + + +def run(tmp_path, *, user=None, **kwargs): + path = tmp_path / "user" / "config.yaml" + path.parent.mkdir(exist_ok=True) + if user is not None or not path.exists(): + path.write_text(yaml.safe_dump(user or {})) + config = Config(config_path=str(path)) + auth = AuthManager(config) + cmd = LoginCommand() + result = cmd.handle(config, auth, DeepgramClient(config, auth), **kwargs) + return cmd, result, path + + +def payload(result): + """The JSON a run writes to stdout for ``result``.""" + return json.loads(json.dumps(result.model_dump())) + + +# ── No key-prefix heuristic ────────────────────────────────────── + + +@pytest.mark.parametrize("key", [HEX_KEY, SK_KEY]) +def test_any_key_the_api_accepts_logs_in( + tmp_path, verified, keyring_store, key, capsys +): + _, result, _ = run(tmp_path, api_key=key) + assert result.status == "success" + assert verified == [key] + assert keyring_store == {"api-key.default": key} + captured = capsys.readouterr() + assert "format" not in captured.out + captured.err + assert key not in captured.out + captured.err + + +def test_auth_manager_has_no_prefix_warning(tmp_path, verified, capsys): + config = Config(config_path=str(tmp_path / "c.yaml")) + AuthManager(config).login_with_api_key(HEX_KEY) + captured = capsys.readouterr() + assert "pattern" not in captured.out + captured.err + + +# ── The env-key and re-login traps ─────────────────────────────── + + +def test_env_key_with_api_key_flag_is_confirmation_required( + tmp_path, verified, keyring_store, monkeypatch, capsys +): + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + monkeypatch.setattr(sys, "argv", ["dg", "login", "--api-key", HEX_KEY]) + cmd, result, path = run(tmp_path, api_key=HEX_KEY) + assert isinstance(result, LoginCodedError) + assert payload(result) == { + "error": ( + "DEEPGRAM_API_KEY is set, and logging in saves a profile that dg will " + "use instead of it." + ), + "status": "error", + "code": "confirmation_required", + # The key is on argv, so no runnable command can be given. + "hint": ( + "This job's DEEPGRAM_API_KEY is used without a login. Drop the login " + "step, or add --yes if a saved profile is intended." + ), + "next_step": None, + "schema_version": 1, + "details": {"flag": "--yes"}, + } + assert list(payload(result)) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + "details", + ] + assert cmd.exit_code_for(result) == 1 + assert verified == [] and keyring_store == {} + assert yaml.safe_load(path.read_text()) == {} + err = capsys.readouterr().err + assert err.startswith("ERROR: DEEPGRAM_API_KEY is set") + assert "HINT: This job's DEEPGRAM_API_KEY" in err + assert HEX_KEY not in err + + +def test_env_key_bare_login_names_login_yes(tmp_path, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + monkeypatch.setattr(login_module, "is_ai_agent", lambda: True) + _, result, _ = run(tmp_path) + assert result.code == "confirmation_required" + assert result.next_step == "dg login --yes" + assert result.details == {"flag": "--yes"} + assert result.hint == ( + "Run next_step only if the user agrees: the saved login will then take " + "precedence over DEEPGRAM_API_KEY." + ) + + +def saved_login(tmp_path): + return {"profiles": {"default": {"api_key": "saved-key-0000"}}} + + +def test_relogin_is_confirmation_required(tmp_path, verified, monkeypatch): + monkeypatch.setattr( + sys, "argv", ["dg", "-o", "json", "login", "--profile", "default"] + ) + _, result, path = run(tmp_path, user=saved_login(tmp_path), profile="default") + assert result.code == "confirmation_required" + assert result.error == 'Profile "default" already has a saved login.' + assert result.next_step == "dg -o json login --profile default --yes" + assert result.details == {"flag": "--yes"} + assert result.hint == "Run the command in next_step only if the user agrees." + assert verified == [] + assert yaml.safe_load(path.read_text()) == saved_login(tmp_path) + + +def test_relogin_with_key_on_argv_has_no_next_step(tmp_path, verified, monkeypatch): + monkeypatch.setattr(login_module, "is_ai_agent", lambda: True) + monkeypatch.setattr(sys, "argv", ["dg", "login", "-k", HEX_KEY]) + _, result, _ = run(tmp_path, user=saved_login(tmp_path), api_key=HEX_KEY) + assert result.next_step is None + assert result.hint == ( + "Ask the user to rerun the command with --yes in their own terminal if " + "they agree." + ) + assert HEX_KEY not in json.dumps(payload(result)) + + +def test_human_without_a_terminal_is_told_to_add_yes(tmp_path, monkeypatch, capsys): + output._output_config.update(agentic=False, format="default") + _, result, _ = run(tmp_path, user=saved_login(tmp_path)) + assert result.code == "confirmation_required" + assert result.hint == "To go ahead, run the command again with --yes." + err = capsys.readouterr().err + assert "already has a saved login" in err + + +@pytest.mark.parametrize("flag", ["yes", "force_write"]) +def test_yes_and_force_write_answer_both_traps( + tmp_path, verified, keyring_store, monkeypatch, flag +): + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + _, result, _ = run( + tmp_path, user=saved_login(tmp_path), api_key=HEX_KEY, **{flag: True} + ) + assert result.status == "success" + assert keyring_store == {"api-key.default": HEX_KEY} + + +def test_no_trap_ever_returns_cancelled(tmp_path, monkeypatch): + """Ledger 18: without a person saying No, nothing is `cancelled`.""" + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + for user in ({}, saved_login(tmp_path)): + _, result, _ = run(tmp_path, user=user, api_key=HEX_KEY) + assert result.status == "error" and result.code == "confirmation_required" + + +def test_a_person_still_gets_the_prompt(tmp_path, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + asked: list[str] = [] + + def confirm(self, message, default=False): + asked.append(message) + return False + + monkeypatch.setattr(LoginCommand, "confirm", confirm) + monkeypatch.setattr(LoginCommand, "_can_ask", lambda self: True) + _, result, _ = run(tmp_path, api_key=HEX_KEY) + assert asked == ["Do you still want to login with a profile?"] + assert result.status == "cancelled" # the person said No + + +def test_can_ask_needs_a_person_at_a_terminal(monkeypatch): + cmd = LoginCommand() + monkeypatch.setattr(login_module, "_interactive_terminal", lambda: True) + monkeypatch.setattr(LoginCommand, "can_prompt", lambda self: True) + assert cmd._can_ask() is True + monkeypatch.setattr(login_module, "is_ai_agent", lambda: True) + assert cmd._can_ask() is False + monkeypatch.setattr(login_module, "is_ai_agent", lambda: False) + monkeypatch.setattr(login_module, "_interactive_terminal", lambda: False) + assert cmd._can_ask() is False + + +# ── next_step construction ─────────────────────────────────────── + + +@pytest.mark.parametrize( + "argv", + [ + ["login", "--api-key", "K"], + ["login", "--api-key=K"], + ["login", "-k", "K"], + ["login", "-kK"], + ["--api-key", "K", "login"], + ["--base-url", "https://user:pw@example.com", "login"], + ["--base-url", "https://example.com/?token=1", "login"], + ], +) +def test_secret_bearing_argv_has_no_next_step(monkeypatch, argv): + monkeypatch.setattr(sys, "argv", ["dg", *argv]) + assert login_module._secret_in(argv) is True + assert login_module._rerun_with_yes("dg") is None + + +def test_rerun_quotes_and_refuses_unsafe_tokens(monkeypatch): + monkeypatch.setattr(sys, "argv", ["dg", "login", "--profile", "my work"]) + assert login_module._rerun_with_yes("dg") == "dg login --profile 'my work' --yes" + monkeypatch.setattr(sys, "argv", ["dg", "login", "--profile", "=x"]) + assert login_module._rerun_with_yes("dg") is None + monkeypatch.setattr(sys, "argv", ["dg", "login"]) + assert login_module._rerun_with_yes(None) is None + + +def test_details_is_left_out_when_empty(): + err = LoginCodedError(error="e", code="c", hint="h", next_step=None) + assert list(err.model_dump()) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + ] diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index f9d3c7ff..42ae5da3 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -575,12 +575,6 @@ def login_with_api_key( project_id: Deepgram project ID (optional — can be set later) force_write: Skip confirmation prompts """ - # Validate API key format (basic check) - if not api_key.startswith(("sk-", "pk-")): - console.print( - "[red]Warning:[/red] API key format doesn't match expected pattern" - ) - # Verify API key (and project if provided) console.print("[dim]Verifying credentials...[/dim]") success, message, error_type = self.verify_credentials( diff --git a/packages/deepctl-core/src/deepctl_core/base_command.py b/packages/deepctl-core/src/deepctl_core/base_command.py index ca6af5b9..14830062 100644 --- a/packages/deepctl-core/src/deepctl_core/base_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_command.py @@ -479,6 +479,18 @@ def _output_csv(self, result: Any) -> None: # Fallback to JSON self._output_json(result) + def can_prompt(self) -> bool: + """Whether this run may ask a question at all. + + False in an agentic or non-interactive context, for a command that + isn't CI-friendly, and when the command was given any input (see + ``is_guided``). A command that must not fall back to a default + answers an unanswerable prompt itself (``confirmation_required``). + """ + return not ( + _agentic or not self.ci_friendly or not getattr(self, "_guided", True) + ) + def confirm(self, message: str, default: bool = False) -> bool: """Ask for user confirmation. @@ -487,9 +499,10 @@ def confirm(self, message: str, default: bool = False) -> bool: default: Default value if user just presses Enter Returns: - True if confirmed, False otherwise + True if confirmed, False otherwise; ``default`` when + ``can_prompt()`` is False """ - if _agentic or not self.ci_friendly or not getattr(self, "_guided", True): + if not self.can_prompt(): return default try: diff --git a/packages/deepctl-core/src/deepctl_core/messages.py b/packages/deepctl-core/src/deepctl_core/messages.py index 73a60597..18618905 100644 --- a/packages/deepctl-core/src/deepctl_core/messages.py +++ b/packages/deepctl-core/src/deepctl_core/messages.py @@ -134,6 +134,34 @@ E_PERSON_ONLY_AGENT_HINT = ( "A person must run this command. If you are an AI agent, ask the user." ) +# A login prompt that can't be shown (catalogue strings). Hints: +# H = person, A = AI agent, C = automation (CI, cron); *_NULL when next_step +# is null. {prog} is the running executable. +E_CONFIRM_ENVKEY = ( + "DEEPGRAM_API_KEY is set, and logging in saves a profile that {prog} will " + "use instead of it." +) +A_CONFIRM_ENVKEY = ( + "Run next_step only if the user agrees: the saved login will then take " + "precedence over DEEPGRAM_API_KEY." +) +A_CONFIRM_ENVKEY_NULL = ( + "Ask the user whether to save a login profile; {prog} will then use it " + "instead of DEEPGRAM_API_KEY. Never put a key on the command line." +) +C_CONFIRM_ENVKEY_NULL = ( + "This job's DEEPGRAM_API_KEY is used without a login. Drop the login step, " + "or add --yes if a saved profile is intended." +) +E_CONFIRM_RELOGIN = 'Profile "{profile}" already has a saved login.' +H_CONFIRM_YES = "To go ahead, run the command again with --yes." +A_CONFIRM_YES = "Run the command in next_step only if the user agrees." +A_CONFIRM_YES_NULL = ( + "Ask the user to rerun the command with --yes in their own terminal if they agree." +) +C_CONFIRM_YES_NULL = ( + "Add --yes to this command in the job if replacing the saved login is intended." +) E_SAVE_CUSTOM_CONFIG = ( "--save-base-url can't be used with --config: only the default user " "config ({user_config}) can approve API hosts. Run the command without " diff --git a/src/deepctl/_manifest.d/login.json b/src/deepctl/_manifest.d/login.json index 4f01b621..700e53ed 100644 --- a/src/deepctl/_manifest.d/login.json +++ b/src/deepctl/_manifest.d/login.json @@ -78,7 +78,29 @@ "is_flag": true, "count": false, "hidden": false, - "help": "Don't prompt for confirmation when providing credentials" + "help": "Same as --yes" + }, + { + "class": "option", + "name": "yes", + "opts": [ + "--yes", + "-y" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Save without asking, even if DEEPGRAM_API_KEY is set or the profile already has a saved login (--save-base-url still asks)" }, { "class": "option", From 1f4dc3772ea207b0496d0058f000e8112c66bdd7 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 01:16:25 -0400 Subject: [PATCH 02/18] feat(login): let agents finish device login with --complete In an AI agent session (or with DG_AGENT=1, --no-browser, or neither stdin nor stdout a terminal), dg login now asks for a device code, saves it and returns at once with the link and code (status "pending", exit 0). dg login --complete checks once: auth_pending with details.retry_after until the user approves, then saves the key as before; auth_denied or auth_expired otherwise, and auth_expired with next_step "dg login" when no login is waiting. --wait keeps 0.4.0's behavior of polling until done; a person at a terminal still gets it by default. The pending code lives in {state_dir}/login-pending.json: written 0600 through O_CREAT|O_EXCL|O_NOFOLLOW and an atomic replace, in a 0700 directory, holding the device code, interval, expiry, login host and profile, never a key. state_dir() is ~/.deepctl, the one state-dir helper later changes reuse. In an AI agent session the file must be owned by this user with no group or other write bit; otherwise it is ignored with one WARN: line and left in place. A code is only sent back to the login host that issued it. Before any request, login checks that it could save the result: an unwritable state or config directory is state_unwritable (E-STATE-UNWRITABLE); in an AI agent session, a user config dg won't save over is state_unwritable with details.reason "untrusted_config". Both come before the --yes confirmations, and both apply to --api-key and --complete too. A home dg can't write no longer stops every command while Config picks its default path. The token poll is a POST to the RFC 8628 token endpoint, so the device code never lands in a URL, and a fixed device name replaces this machine's hostname. Polls are at least 6 s apart whatever the server asks; slow_down, 429 and server errors back off by 5 s or the server's Retry-After, and --wait keeps polling through them until the code expires. A device-code answer that isn't a JSON object is network_unavailable. The login link must be https (or loopback http). Status lines go to stderr. In mode A they are dropped, and the link and code are an INFO: line; a person's terminal shows the same line instead of the separate code and link lines. --complete refuses --profile for another profile than the pending login's, and --save-base-url. The legacy config migration prints to stderr. --- .../src/deepctl_cmd_login/command.py | 578 +++++++++++++- .../src/deepctl_cmd_login/models.py | 11 + .../tests/unit/test_login_base_url.py | 1 + .../tests/unit/test_login_command.py | 10 +- .../tests/unit/test_login_device_agent.py | 715 ++++++++++++++++++ .../tests/unit/test_login_strings.py | 79 ++ .../deepctl-core/src/deepctl_core/auth.py | 275 +++++-- .../deepctl-core/src/deepctl_core/config.py | 21 +- .../src/deepctl_core/login_state.py | 307 ++++++++ .../deepctl-core/src/deepctl_core/messages.py | 67 ++ packages/deepctl-core/tests/unit/test_auth.py | 172 ++++- .../tests/unit/test_config_login.py | 36 + .../tests/unit/test_login_state.py | 186 +++++ src/deepctl/_manifest.d/login.json | 65 +- 14 files changed, 2397 insertions(+), 126 deletions(-) create mode 100644 packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py create mode 100644 packages/deepctl-cmd-login/tests/unit/test_login_strings.py create mode 100644 packages/deepctl-core/src/deepctl_core/login_state.py create mode 100644 packages/deepctl-core/tests/unit/test_config_login.py create mode 100644 packages/deepctl-core/tests/unit/test_login_state.py diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index 820c12ff..0f073e7e 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1,10 +1,15 @@ """Login command for deepctl.""" +import dataclasses import os import shlex import shutil import sys -from typing import Any +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, TextIO from urllib.parse import urlsplit import click @@ -23,6 +28,8 @@ is_ai_agent, ) from deepctl_core import messages as msg +from deepctl_core.auth import DeviceCodeUnreadable +from deepctl_core.config import ConfigSaveRefused from deepctl_core.hosts import ( SOURCE_ENV, SOURCE_FLAG, @@ -37,21 +44,44 @@ report, user_config_display, ) +from deepctl_core.login_state import ( + MIN_POLL_INTERVAL, + LoadedPending, + PendingLogin, + load_pending, + remove_pending, + save_pending, + state_dir, + unwritable_dir, +) from deepctl_core.output import ( _output_config, print_error, print_error_hint, print_hint, print_info, + print_security_warning, print_warning, + stderr_console, ) from rich.console import Console from rich.markup import escape from rich.prompt import Prompt -from .models import LoginCodedError, LoginResult, LogoutResult, WhoamiResult +if TYPE_CHECKING: + from deepctl_core.auth import DeviceCodeResponse + +from .models import ( + LoginCodedError, + LoginPendingResult, + LoginResult, + LogoutResult, + WhoamiResult, +) console = Console() +# Status chrome of the login flows: stderr, never the payload's stdout. +status = stderr_console _PROG_NAMES = ("dg", "deepgram", "deepctl") _YES_FLAG = "--yes" @@ -129,6 +159,11 @@ def _rerun_with_yes(prog: str | None) -> str | None: return step if shlex.join(shlex.split(step)) == step else None +def _hard_signal() -> bool: + """An AI agent marked the environment, or ``DG_AGENT=1`` says one runs dg.""" + return is_ai_agent() or os.environ.get("DG_AGENT") == "1" + + def _hint_for( human: str, agent: str, @@ -138,7 +173,7 @@ def _hint_for( automation_null: str | None = None, ) -> str: """The hint for this run's mode, null-aware.""" - if is_ai_agent(): + if _hard_signal(): return agent_null if next_step is None and agent_null else agent if _output_config["agentic"]: if next_step is None: @@ -162,6 +197,145 @@ def _coded_error( ) +# Mode A stderr grammar: every line a caller reads starts with +# one of these. Anything else a login flow prints is status chrome. +_MODE_A_PREFIXES = ( + "ERROR: ", + "WARN: ", + "INFO: ", + "HINT: ", + "OK: ", + "DEPRECATED: ", +) + + +class _ModeAStderr: + """``sys.stderr`` in mode A: each write that starts a line is kept or + dropped as a whole. With ``verbose``, chrome is kept as ``INFO:`` lines + instead of dropped.""" + + def __init__(self, stream: TextIO, verbose: bool = False) -> None: + self._stream = stream + self._verbose = verbose + self._keep = True + self._line_start = True + + def write(self, text: str) -> int: + if not text: + return 0 + if self._line_start: + first = text.lstrip("\n").split("\n", 1)[0] + prefixed = first.startswith(_MODE_A_PREFIXES) + self._keep = bool(first.strip()) and (prefixed or self._verbose) + if self._keep and not prefixed: + lines = [line for line in text.strip("\n").split("\n") if line.strip()] + text = "".join(f"INFO: {line.strip()}\n" for line in lines) + if self._keep: + self._stream.write(text) + self._line_start = text.endswith("\n") + return len(text) + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + +@contextmanager +def _without_chrome() -> Iterator[None]: + """In mode A, drop status chrome from stderr (``INFO:`` lines with + --verbose).""" + if not _output_config["agentic"]: + yield + return + real = sys.stderr + sys.stderr = _ModeAStderr(real, verbose=bool(_output_config["verbose"])) + try: + yield + finally: + sys.stderr = real + + +def _shown_prog(prog: str | None) -> str: + return prog or _PROG_NAMES[0] + + +def _state_unwritable(directory: Path) -> LoginCodedError: + shown = _shown_prog(_prog()) + return _coded_error( + msg.E_STATE_UNWRITABLE.format(prog=shown, dir=msg.display_path(directory)), + msg.CODE_STATE_UNWRITABLE, + _hint_for( + msg.H_STATE_UNWRITABLE.format(prog=shown), + msg.A_STATE_UNWRITABLE.format(prog=shown), + None, + ), + None, + {"dir": str(directory)}, + ) + + +def _preflight(config: Config) -> Any | None: + """Stop before any request when the login couldn't be saved. + + In order: the state or credential directory isn't writable + (``E-STATE-UNWRITABLE``); then, in an AI agent session, the user config + is one dg won't save over (``E-CFG-SAVE-UNTRUSTED``). Both + come before the confirmations, so an agent is never told to pass --yes + for a save that would fail. + """ + config_path = Path(config.config_path) + for directory, file in ( + (state_dir(), None), + (config_path.parent, config_path), + ): + bad = unwritable_dir(directory, file) + if bad is not None: + return _state_unwritable(bad) + try: + config.check_saveable() + except ConfigSaveRefused as e: + shown = _shown_prog(_prog()) + return _coded_error( + msg.E_CFG_SAVE_UNTRUSTED.format( + prog=shown, + path=msg.display_path(config_path), + why=msg.W_CFG_UNTRUSTED_WHY[e.why], + ), + msg.CODE_STATE_UNWRITABLE, + msg.W_CFG_UNTRUSTED_FILE_HINT, + None, + {"reason": "untrusted_config", "path": str(config_path)}, + ) + except OSError as e: + # A save that can't happen for another reason (not a regular file, + # content that wouldn't load back): stop before sending anything. + print_error(escape(f"Login failed: {e}")) + print_error_hint(e) + return LoginResult( + status="error", + message=str(e), + profile=config.profile or "default", + api_key_masked=None, + ) + return None + + +def _device_code_unreadable(host: str) -> LoginCodedError: + """The identity provider's device-code answer couldn't be read.""" + shown = _shown_prog(_prog()) + values = {"host": msg.display(host), "prog": shown} + return _coded_error( + msg.E_NETWORK_UNREADABLE.format(**values), + msg.CODE_NETWORK_UNAVAILABLE, + _hint_for( + msg.H_NETWORK_UNREADABLE.format(**values), + msg.A_NETWORK_UNREADABLE, + None, + ), + None, + {"host": msg.display(host), "reason": "unreadable response"}, + ) + + def _confirmation_required(error: str, next_step: str | None, hint: str) -> Any: return _coded_error( error, @@ -190,16 +364,18 @@ class LoginCommand(BaseCommand): examples = [ "dg login", + "dg login --no-browser", + "dg login --complete", "dg login --api-key YOUR_API_KEY", "dg login --api-key YOUR_KEY --project-id YOUR_PROJECT", "dg login --profile staging --api-key YOUR_KEY", "dg --base-url https://api.eu.deepgram.com login --save-base-url", ] agent_help = ( - "Authenticate with Deepgram. Can accept an API key directly via " - "--api-key flag, or prompt interactively. Credentials are stored " - "securely in the system keyring. Supports named profiles for " - "managing multiple environments." + "Log in to Deepgram. In an AI agent session, dg login prints a link " + "and code and exits: give both to the user, then run dg login " + "--complete (add --wait to keep checking). Never put an API key on " + "the command line. Profiles keep separate API keys and project IDs." ) def get_arguments(self) -> list[dict[str, Any]]: @@ -241,6 +417,35 @@ def get_arguments(self) -> list[dict[str, Any]]: "required": False, "is_option": True, }, + { + "names": ["--no-browser"], + "help": ( + "Print the login link and code instead of opening a browser, " + "then exit; finish with --complete, or add --wait to wait here" + ), + "is_flag": True, + "is_option": True, + }, + { + "names": ["--complete"], + "help": ( + "Finish a pending browser login: check once whether it was " + "approved, and save the key if it was (add --wait to keep " + "checking)" + ), + "is_flag": True, + "is_option": True, + }, + { + "names": ["--wait"], + "help": ( + "Wait until the browser login is approved (the default in a " + "terminal), instead of exiting with the link and code; with " + "--complete, keep checking" + ), + "is_flag": True, + "is_option": True, + }, { "names": ["--save-base-url"], "help": ( @@ -260,11 +465,18 @@ def handle( **kwargs: Any, ) -> Any: """Handle login command.""" + with _without_chrome(): + return self._login(config, auth_manager, **kwargs) + + def _login(self, config: Config, auth_manager: AuthManager, **kwargs: Any) -> Any: api_key = kwargs.get("api_key") project_id = kwargs.get("project_id") force_write = bool(kwargs.get("yes") or kwargs.get("force_write")) profile = kwargs.get("profile") save_base_url = kwargs.get("save_base_url", False) + complete = bool(kwargs.get("complete")) + wait = bool(kwargs.get("wait")) + no_browser = bool(kwargs.get("no_browser")) # Set profile if provided if profile: @@ -272,6 +484,18 @@ def handle( current_profile = config.profile or "default" + if complete: + if api_key: + raise click.UsageError("--complete can't be used with --api-key.") + if save_base_url: + raise click.UsageError("--complete can't be used with --save-base-url.") + return self._complete(config, auth_manager, wait=wait) + + # Nothing is asked or sent when the login couldn't be saved. + stop = _preflight(config) + if stop is not None: + return stop + # Decide where the key may go before doing anything else. A base URL # from --base-url or DEEPGRAM_BASE_URL is used for this login only; # --save-base-url, confirmed by a person, is the one way to save it. @@ -313,21 +537,21 @@ def handle( automation_null=msg.C_CONFIRM_ENVKEY_NULL, ), ) - console.print( + status.print( "[yellow]Environment variable DEEPGRAM_API_KEY is set.[/yellow]" ) if has_env_project: - console.print( + status.print( "[yellow]Environment variable DEEPGRAM_PROJECT_ID is also set.[/yellow]\n" "[green]Commands will work using these environment variables.[/green]" ) else: - console.print( + status.print( "[yellow]Note: DEEPGRAM_PROJECT_ID is not set. " "You'll need to set it or provide it via --project-id flag.[/yellow]" ) - console.print( + status.print( "\n[dim]Logging in will create a profile that takes precedence " "over environment variables.[/dim]" ) @@ -361,7 +585,7 @@ def handle( automation_null=msg.C_CONFIRM_YES_NULL, ), ) - console.print( + status.print( f"[yellow]Already logged in to profile:[/yellow] {current_profile}" ) @@ -398,7 +622,9 @@ def handle( config, auth_manager, str(api_key), project_id, force_write ) else: - result = self._web_auth(config, auth_manager, force_write) + result = self._web_auth( + config, auth_manager, wait=wait, no_browser=no_browser + ) if result.status == "success": self._report_base_url(config, current_profile) @@ -409,6 +635,16 @@ def handle( return result + def _pending_mode(self, *, wait: bool, no_browser: bool) -> bool: + """Return the link and code at once instead of waiting: + under a hard agent signal, with --no-browser, or when neither stdin + nor stdout is a terminal. --wait always waits.""" + if wait: + return False + if no_browser or _hard_signal(): + return True + return not sys.stdin.isatty() and not sys.stdout.isatty() + def _can_ask(self) -> bool: """A person can answer login's confirmations here.""" return self.can_prompt() and _interactive_terminal() and not is_ai_agent() @@ -648,18 +884,19 @@ def _cli_auth( force_write: bool, ) -> LoginResult: """Handle CLI authentication with API key.""" - console.print("[blue]Configuring CLI with API key...[/blue]") + status.print("[blue]Configuring CLI with API key...[/blue]") # The key's format isn't checked: Deepgram keys have no fixed prefix, # and login_with_api_key verifies the key with the API. # Validate project ID is provided with API key if not project_id: - console.print("[yellow]Warning:[/yellow] Project ID not provided") - console.print( - "You can set it later with: deepctl login --project-id " + status.print("[yellow]Warning:[/yellow] Project ID not provided") + status.print( + "You can set it later with: " + f"{_shown_prog(_prog())} login --project-id " ) - console.print("Or use environment variable: DEEPGRAM_PROJECT_ID") + status.print("Or use environment variable: DEEPGRAM_PROJECT_ID") # No need to check for existing config - we handle that in the main flow @@ -684,7 +921,7 @@ def _cli_auth( ) except AuthenticationError as e: - console.print(f"[red]Login failed:[/red] {e}") + print_error(escape(f"Login failed: {e}")) return LoginResult( status="error", message=str(e), @@ -693,7 +930,7 @@ def _cli_auth( ) except Exception as e: - console.print(f"[red]Error:[/red] {e}") + print_error(escape(f"Login failed: {e}")) print_error_hint(e) return LoginResult( status="error", @@ -703,23 +940,38 @@ def _cli_auth( ) def _web_auth( - self, config: Config, auth_manager: AuthManager, force_write: bool - ) -> LoginResult | CodedErrorResult: + self, + config: Config, + auth_manager: AuthManager, + *, + wait: bool = False, + no_browser: bool = False, + ) -> Any: """Handle web authentication with device flow.""" # The identity host mints the key: check it before anything else, and # keep a refusal off stdout apart from the structured result. try: - check_identity_host(config) + identity_url = check_identity_host(config) except CredentialHostError as e: return CodedErrorResult(error=e.error, code=e.code, hint=e.hint) - console.print("[blue]Starting web authentication...[/blue]") + if self._pending_mode(wait=wait, no_browser=no_browser): + return self._start_pending(config, auth_manager, identity_url) - # No need to check for existing config - we handle that in the main flow + status.print("[blue]Starting web authentication...[/blue]") try: - # Start device flow - auth_manager.login_with_device_flow() + open_browser = not (no_browser or _hard_signal()) + if _output_config["agentic"]: + # Mode A: the link and code as an INFO line; chrome is dropped. + device = auth_manager._request_device_code(identity_url) + self._announce(device) + if open_browser: + _open_browser(device) + token = auth_manager._poll_for_token(device, identity_url) + auth_manager._store_token(token) + else: + auth_manager.login_with_device_flow(open_browser=open_browser) profile_name = config.profile or "default" @@ -745,8 +997,11 @@ def _web_auth( config_path=str(config.config_path), ) + except DeviceCodeUnreadable as e: + return _device_code_unreadable(e.host) + except AuthenticationError as e: - console.print(f"[red]Authentication failed:[/red] {e}") + print_error(escape(f"Login failed: {e}")) return LoginResult( status="error", message=str(e), @@ -755,7 +1010,7 @@ def _web_auth( ) except KeyboardInterrupt: - console.print("\n[yellow]Authentication cancelled by user[/yellow]") + status.print("\n[yellow]Authentication cancelled by user[/yellow]") return LoginResult( status="cancelled", message="Login cancelled by user", @@ -764,7 +1019,7 @@ def _web_auth( ) except Exception as e: - console.print(f"[red]Error during web authentication:[/red] {e}") + print_error(escape(f"Error during web authentication: {e}")) print_error_hint(e) return LoginResult( status="error", @@ -773,6 +1028,269 @@ def _web_auth( api_key_masked=None, ) + def _announce(self, device: "DeviceCodeResponse") -> None: + """Print the link and code as an INFO line (stderr).""" + print_info( + escape( + msg.I_LOGIN_PENDING.format( + verification_uri=msg.display(device.verification_uri), + user_code=msg.display(device.user_code or ""), + ) + ), + stderr=True, + ) + + def _start_pending( + self, config: Config, auth_manager: AuthManager, identity_url: str + ) -> Any: + """Get a device code, save it for --complete, and return the link and + code at once (exit 0; OQ-16).""" + profile_name = config.profile or "default" + try: + device = auth_manager._request_device_code(identity_url) + if not device.user_code: + raise AuthenticationError( + "the login service didn't return a code. Try again in a moment." + ) + except DeviceCodeUnreadable as e: + return _device_code_unreadable(e.host) + except AuthenticationError as e: + print_error(escape(f"Login failed: {e}")) + return LoginResult( + status="error", + message=str(e), + profile=profile_name, + api_key_masked=None, + ) + pending = PendingLogin( + device_code=device.device_code, + user_code=device.user_code, + verification_uri=device.verification_uri, + interval=max(MIN_POLL_INTERVAL, device.interval), + expires_at=time.time() + device.expires_in, + host=identity_url, + profile=profile_name, + ) + try: + save_pending(pending) + except OSError: + return _state_unwritable(state_dir()) + + prog = _prog() + shown = _shown_prog(prog) + next_step = f"{prog} login --complete" if prog else None + self._announce(device) + hint = ( + msg.A_LOGIN_PENDING + if _hard_signal() or _output_config["agentic"] + else msg.H_LOGIN_PENDING + ) + print_hint(escape(hint.format(prog=shown))) + return LoginPendingResult( + message=msg.M_LOGIN_PENDING, + verification_uri=device.verification_uri, + user_code=device.user_code, + expires_in=device.expires_in, + next_step=next_step, + ) + + def _complete( + self, config: Config, auth_manager: AuthManager, *, wait: bool + ) -> Any: + """``dg login --complete``: poll once (or, with --wait, until done) + for the saved device code, and save the key once it is approved.""" + import httpx + + stop = _preflight(config) + if stop is not None: + return stop + prog = _prog() + shown = _shown_prog(prog) + loaded = load_pending(agent=is_ai_agent()) + if loaded.ignored_why is not None: + print_security_warning( + escape( + msg.W_LOGIN_PENDING_UNTRUSTED.format( + path=msg.display_path(loaded.path), + prog=shown, + why=msg.W_CFG_UNTRUSTED_WHY[loaded.ignored_why], + ) + ) + ) + pending = loaded.pending + if pending is None: + return self._auth_expired(prog) + try: + identity_url = check_identity_host(config) + except CredentialHostError as e: + return CodedErrorResult(error=e.error, code=e.code, hint=e.hint) + if canonical_url(pending.host) != identity_url: + # Issued by another login host: never send its code elsewhere. + return self._auth_expired(prog) + + explicit = getattr(config, "_explicit_profile", None) + if explicit and explicit != pending.profile: + # The pending login was confirmed for its own profile. + raise click.UsageError( + f'The pending login is for profile "{msg.display(pending.profile)}". ' + "Run --complete without --profile, or start a new login for " + f'"{msg.display(explicit)}" with dg login --profile.' + ) + config.profile = pending.profile + interval = pending.interval + while True: + if pending.expired(): + remove_pending(loaded) + return self._auth_expired(prog) + try: + poll = auth_manager.poll_device_token(pending.device_code, identity_url) + except httpx.RequestError as e: + text = f"Network error while completing the login: {e}" + print_error(escape(text)) + return LoginResult( + status="error", + message=text, + profile=pending.profile, + api_key_masked=None, + ) + if poll.state == "token" and poll.token is not None: + return self._finish(config, auth_manager, loaded, poll.token) + if poll.state == "unavailable" and not wait: + text = ( + f"Couldn't complete the login: {poll.detail}. " + f"Run {shown} login to start again." + ) + print_error(escape(text)) + return LoginResult( + status="error", + message=text, + profile=pending.profile, + api_key_masked=None, + ) + if poll.state in ("pending", "slow_down", "unavailable"): + if poll.state in ("slow_down", "unavailable"): + interval = max(interval + 5, poll.retry_after or 0) + pending = dataclasses.replace(pending, interval=interval) + try: + save_pending(pending) + loaded = load_pending(agent=is_ai_agent()) + except OSError: + pass # the next --complete waits the old interval + if wait: + time.sleep( + max(0.0, min(interval, pending.expires_at - time.time())) + ) + continue + return self._auth_pending(prog, pending) + if poll.state == "denied": + remove_pending(loaded) + return _coded_error( + msg.E_AUTH_DENIED, + msg.CODE_AUTH_DENIED, + _hint_for( + msg.H_AUTH_DENIED.format(prog=shown), + msg.A_AUTH_DENIED, + None, + ), + None, + ) + if poll.state == "expired": + remove_pending(loaded) + return self._auth_expired(prog) + text = ( + f"Couldn't complete the login: {poll.detail}. " + f"Run {shown} login to start again." + ) + print_error(escape(text)) + return LoginResult( + status="error", + message=text, + profile=pending.profile, + api_key_masked=None, + ) + + def _finish( + self, + config: Config, + auth_manager: AuthManager, + loaded: LoadedPending, + token: Any, + ) -> Any: + """Save the approved key (keyring only once the save can happen).""" + profile_name = config.profile or "default" + try: + auth_manager._store_token(token) + config._config.active_profile = profile_name + config.save() + except Exception as e: + print_error(escape(f"Login failed: {e}")) + print_error_hint(e) + return LoginResult( + status="error", + message=str(e), + profile=profile_name, + api_key_masked=None, + ) + finally: + # The code is used up either way. + remove_pending(loaded) + key = token.dg_token or "" + return LoginResult( + status="success", + message="Successfully logged in via web authentication", + profile=profile_name, + api_key_masked=f"****{key[-4:]}" if key else None, + project_id=token.project_id or token.pid, + config_path=str(config.config_path), + ) + + def _auth_expired(self, prog: str | None) -> LoginCodedError: + shown = _shown_prog(prog) + return _coded_error( + msg.E_AUTH_EXPIRED, + msg.CODE_AUTH_EXPIRED, + _hint_for( + msg.H_AUTH_EXPIRED.format(prog=shown), + msg.A_AUTH_EXPIRED.format(prog=shown), + f"{prog} login" if prog else None, + ), + f"{prog} login" if prog else None, + ) + + def _auth_pending(self, prog: str | None, pending: PendingLogin) -> LoginCodedError: + shown = _shown_prog(prog) + values = { + "prog": shown, + "retry_after": pending.interval, + "verification_uri": msg.display(pending.verification_uri), + "user_code": msg.display(pending.user_code), + } + next_step = f"{prog} login --complete" if prog else None + return _coded_error( + msg.E_AUTH_PENDING, + msg.CODE_AUTH_PENDING, + _hint_for( + msg.H_AUTH_PENDING.format(**values), + msg.A_AUTH_PENDING.format(**values), + next_step, + ), + next_step, + { + "retry_after": pending.interval, + "verification_uri": pending.verification_uri, + "user_code": pending.user_code, + }, + ) + + +def _open_browser(device: "DeviceCodeResponse") -> None: + import webbrowser + + try: + webbrowser.open(device.verification_uri_complete or device.verification_uri) + except Exception as e: + print_warning(escape(f"Could not open a browser: {e}"), stderr=True) + class LogoutCommand(BaseCommand): """Logout command for clearing authentication.""" diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py index 4355d336..1a7b4aeb 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/models.py @@ -37,6 +37,17 @@ class LoginResult(BaseResult): config_path: str | None = None +class LoginPendingResult(BaseResult): + """``dg login`` handed the device login to a person; exit 0.""" + + status: str = "pending" + verification_uri: str + user_code: str + expires_in: int + next_step: str | None = None + schema_version: int = 1 + + class LogoutResult(BaseResult): """Return structure for logout command.""" diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_base_url.py b/packages/deepctl-cmd-login/tests/unit/test_login_base_url.py index 175518c8..c8b9dcf7 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_base_url.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_base_url.py @@ -68,6 +68,7 @@ def login(tmp_path, *, user=None, flag=None, **kwargs): path = tmp_path / "user" / "config.yaml" path.parent.mkdir(exist_ok=True) path.write_text(yaml.safe_dump(user or {})) + path.chmod(0o600) # as dg writes it; any umask config = Config(config_path=str(path), base_url=flag) auth = AuthManager(config) kwargs.setdefault("api_key", "sk-LOGINKEY") diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_command.py b/packages/deepctl-cmd-login/tests/unit/test_login_command.py index 1a913932..e63fa16d 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_command.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_command.py @@ -51,6 +51,8 @@ def home(tmp_path, monkeypatch): monkeypatch.setattr(con, "_width", 400) saved = dict(output._output_config) output._output_config.update(agentic=True, format="default", quiet=False) + # The mock config has no real file: nothing for login's pre-flight to check. + monkeypatch.setattr(login_module, "_preflight", lambda config: None) yield use_home(monkeypatch, tmp_path / "home") output._output_config.clear() output._output_config.update(saved) @@ -182,10 +184,14 @@ def mock_client(): @pytest.fixture -def login_command(): - """A login command run by a person in a terminal (its prompts can show).""" +def login_command(monkeypatch): + """A login command run by a person in a terminal (its prompts can show, + and the browser login waits), with nothing on disk to check.""" cmd = LoginCommand() cmd._can_ask = lambda: True # type: ignore[method-assign] + cmd._pending_mode = lambda **kw: False # type: ignore[method-assign] + monkeypatch.setattr(login_module, "_preflight", lambda config: None) + monkeypatch.setitem(output._output_config, "agentic", False) return cmd diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py b/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py new file mode 100644 index 00000000..9d6f32d6 --- /dev/null +++ b/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py @@ -0,0 +1,715 @@ +"""Device login for agents: `dg login` returns the link and code at once, +`dg login --complete` finishes it, and nothing is requested when the login +couldn't be saved.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +import time + +import pytest +import yaml +from deepctl_cmd_login import command as login_module +from deepctl_cmd_login.command import LoginCommand +from deepctl_cmd_login.models import LoginCodedError, LoginPendingResult +from deepctl_core import AuthManager, Config, DeepgramClient, hosts, login_state, output +from deepctl_core import auth as auth_module +from deepctl_core.auth import DeviceCodeResponse, DevicePoll, TokenResponse + +# The real request method, before the fixture swaps in the fake. +_REAL_REQUEST = AuthManager._request_device_code + +IDENTITY = "https://id.dx.deepgram.com" +LINK = "https://id.dx.deepgram.com/auth/cli" +CODE = "WDJB-MJHT" +KEY = "dgfinite0000000000000000000000000004567" + + +class FakeIdentity: + """Stands in for the login server: records every request; answers polls in turn.""" + + def __init__(self) -> None: + self.requests: list[str] = [] + self.answers: list[DevicePoll] = [] + self.interval = 5 + + def device_code(self, identity_url=None): + self.requests.append(f"code {identity_url}") + return DeviceCodeResponse( + device_code="device-secret-1", + user_code=CODE, + verification_uri=LINK, + expires_in=900, + interval=self.interval, + ) + + def poll(self, device_code, identity_url=None): + self.requests.append(f"poll {device_code} {identity_url}") + return self.answers.pop(0) if self.answers else DevicePoll("pending") + + +def approved() -> DevicePoll: + return DevicePoll( + "token", + token=TokenResponse( + access_token="jwt", dg_token=KEY, dg_expires_in=3600, pid="proj-1" + ), + ) + + +@pytest.fixture +def agent(monkeypatch): + def set_agent(on: bool) -> None: + for mod in (output, login_module): + monkeypatch.setattr(mod, "is_ai_agent", lambda: on) + output._output_config["agentic"] = on + + return set_agent + + +@pytest.fixture(autouse=True) +def env(tmp_path, monkeypatch, agent): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(login_state, "state_dir", lambda: home / ".deepctl") + monkeypatch.setattr(login_module, "state_dir", lambda: home / ".deepctl") + user = home / ".config" / "deepctl" / "config.yaml" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("deepctl_core.config._default_config_file", lambda: user) + monkeypatch.setattr( + "deepctl_core.config._trust_root_location", lambda: (home, user) + ) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(output, "_hints_shown", set()) + monkeypatch.setattr(login_module, "is_agentic", lambda: False) + monkeypatch.setattr(login_module, "_interactive_terminal", lambda: False) + monkeypatch.setattr(login_module, "_prog", lambda: "dg") + monkeypatch.setattr(sys, "argv", ["dg", "login"]) + for name in ("DEEPGRAM_API_KEY", "DEEPGRAM_PROJECT_ID", "DG_AGENT"): + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv("DEEPGRAM_CLI_BASE_URL", raising=False) + saved = dict(output._output_config) + output._output_config.update(quiet=False, verbose=False, format="json") + agent(True) + monkeypatch.setattr(output.stderr_console, "_width", 400) + keyring: dict[str, str] = {} + monkeypatch.setattr( + auth_module.keyring, + "set_password", + lambda service, name, value: keyring.__setitem__(name, value), + ) + monkeypatch.setattr( + auth_module.keyring, "get_password", lambda service, name: keyring.get(name) + ) + + def delete(service, name): + keyring.pop(name, None) + + monkeypatch.setattr(auth_module.keyring, "delete_password", delete) + fake = FakeIdentity() + monkeypatch.setattr(AuthManager, "_request_device_code", fake.device_code) + monkeypatch.setattr(AuthManager, "poll_device_token", fake.poll) + monkeypatch.setattr(LoginCommand, "_maybe_prompt_skills_setup", lambda self: None) + import webbrowser + + monkeypatch.setattr(webbrowser, "open", lambda *a: pytest.fail("browser")) + yield {"home": home, "user": user, "keyring": keyring, "fake": fake} + output._output_config.clear() + output._output_config.update(saved) + + +def write_user(env, data=None, mode=0o600): + env["user"].parent.mkdir(parents=True, exist_ok=True) + env["user"].write_text(yaml.safe_dump(data or {"output": {"format": "json"}})) + os.chmod(env["user"], mode) + + +def run(env, **kwargs): + config = Config(config_path=str(env["user"])) if env["user"].exists() else Config() + auth = AuthManager(config) + cmd = LoginCommand() + result = cmd.handle(config, auth, DeepgramClient(config, auth), **kwargs) + return cmd, result + + +def pending_path(env): + return env["home"] / ".deepctl" / "login-pending.json" + + +def dumped(result): + return json.loads(json.dumps(result.model_dump())) + + +# ── Bare login returns at once ─────────────────────────────────── + + +def test_agent_login_returns_pending_at_once(env, capsys): + start = time.monotonic() + cmd, result = run(env) + assert time.monotonic() - start < 5 + assert isinstance(result, LoginPendingResult) + assert cmd.exit_code_for(result) == 0 + assert dumped(result) == { + "status": "pending", + "message": "Open the link and enter the code to finish logging in.", + "verification_uri": LINK, + "user_code": CODE, + "expires_in": 900, + "next_step": "dg login --complete", + "schema_version": 1, + } + assert list(dumped(result)) == [ + "status", + "message", + "verification_uri", + "user_code", + "expires_in", + "next_step", + "schema_version", + ] + assert capsys.readouterr().err.splitlines() == [ + f"INFO: To finish logging in, open {LINK} and enter the code {CODE}.", + "HINT: Give the user the link and code, then run dg login --complete after " + "they approve.", + ] + path = pending_path(env) + if sys.platform != "win32": + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700 + saved = json.loads(path.read_text()) + assert saved["device_code"] == "device-secret-1" + assert saved["host"] == IDENTITY and saved["profile"] == "default" + assert env["fake"].requests == [f"code {IDENTITY}"] + + +def test_existing_state_dir_is_made_private(env): + (env["home"] / ".deepctl").mkdir(mode=0o755) + os.chmod(env["home"] / ".deepctl", 0o755) + run(env) + if sys.platform != "win32": + assert stat.S_IMODE((env["home"] / ".deepctl").stat().st_mode) == 0o700 + + +@pytest.mark.parametrize("why", ["no_tty", "no_browser", "dg_agent"]) +def test_pending_without_an_agent_marker(env, agent, monkeypatch, why): + agent(False) + kwargs = {} + if why == "no_tty": + monkeypatch.setattr(sys.stdin, "isatty", lambda: False, raising=False) + monkeypatch.setattr(sys.stdout, "isatty", lambda: False, raising=False) + elif why == "no_browser": + kwargs["no_browser"] = True + else: + monkeypatch.setenv("DG_AGENT", "1") + _, result = run(env, **kwargs) + assert isinstance(result, LoginPendingResult) + + +def test_person_at_a_terminal_still_waits(env, agent, monkeypatch): + agent(False) + output._output_config["format"] = "default" + monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False) + monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False) + calls = [] + monkeypatch.setattr( + AuthManager, + "login_with_device_flow", + lambda self, open_browser=True: calls.append(open_browser), + ) + run(env) + assert calls == [True] + assert not pending_path(env).exists() + + +def test_wait_polls_until_approved(env, monkeypatch): + monkeypatch.setattr(auth_module.time, "sleep", lambda s: None) + env["fake"].answers = [DevicePoll("pending"), approved()] + cmd, result = run(env, wait=True) + assert result.status == "success" + assert env["keyring"] == {"api-key.default": KEY} + assert not pending_path(env).exists() + assert [r.split()[0] for r in env["fake"].requests] == ["code", "poll", "poll"] + + +# ── --complete ─────────────────────────────────────────────────── + + +def test_complete_before_approval_is_auth_pending(env, capsys): + run(env) + capsys.readouterr() + cmd, result = run(env, complete=True) + assert cmd.exit_code_for(result) == 1 + assert dumped(result) == { + "error": "Login is waiting for approval in the browser.", + "status": "error", + "code": "auth_pending", + "hint": ( + f"Wait 6 s for the user to approve the login at {LINK} with the code " + f"{CODE}, then run dg login --complete." + ), + "next_step": "dg login --complete", + "schema_version": 1, + # The server's interval of 5 s is raised to dg's 6 s floor. + "details": {"retry_after": 6, "verification_uri": LINK, "user_code": CODE}, + } + assert pending_path(env).exists() + + +def test_complete_after_approval_saves_the_key(env, capsys): + write_user(env) + run(env) + env["fake"].answers = [approved()] + capsys.readouterr() + cmd, result = run(env, complete=True) + assert dumped(result) == { + "status": "success", + "message": "Successfully logged in via web authentication", + "profile": "default", + "api_key_masked": "****4567", + "project_id": "proj-1", + "config_path": str(env["user"]), + } + captured = capsys.readouterr() + assert captured.err == "" # mode A: no chrome (J1.4) + assert env["keyring"] == {"api-key.default": KEY} + saved = yaml.safe_load(env["user"].read_text()) + assert saved["profiles"]["default"]["project_id"] == "proj-1" + assert saved["profiles"]["default"].get("api_key") is None + assert not pending_path(env).exists() + assert KEY not in captured.out + captured.err + + +def test_complete_saves_to_the_profile_login_started_with(env): + write_user(env) + run(env, profile="work") + env["fake"].answers = [approved()] + _, result = run(env, complete=True) + assert result.profile == "work" + assert env["keyring"] == {"api-key.work": KEY} + + +@pytest.mark.parametrize( + ("answer", "code"), + [("denied", "auth_denied"), ("expired", "auth_expired")], +) +def test_complete_denied_or_expired(env, answer, code): + run(env) + env["fake"].answers = [DevicePoll(answer)] + _, result = run(env, complete=True) + assert result.code == code + assert result.next_step == (None if answer == "denied" else "dg login") + assert not pending_path(env).exists() + + +def test_complete_after_local_expiry(env, monkeypatch): + run(env) + later = time.time() + 1000 + monkeypatch.setattr(login_state.time, "time", lambda: later) + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert [r.split()[0] for r in env["fake"].requests] == ["code"] + + +def test_complete_without_a_pending_login(env, capsys): + cmd, result = run(env, complete=True) + assert dumped(result) == { + "error": "No login is waiting to be completed, or it expired.", + "status": "error", + "code": "auth_expired", + "hint": "Run dg login again and give the user the new link and code.", + "next_step": "dg login", + "schema_version": 1, + } + assert env["fake"].requests == [] + + +def test_complete_never_sends_the_code_to_another_login_host(env, monkeypatch): + run(env) + monkeypatch.setenv("DEEPGRAM_CLI_BASE_URL", "https://id.other.example") + output._output_config["agentic"] = False + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setattr(login_module, "is_ai_agent", lambda: False) + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert [r.split()[0] for r in env["fake"].requests] == ["code"] + + +def test_complete_with_api_key_is_a_usage_error(env): + import click + + with pytest.raises(click.UsageError): + run(env, complete=True, api_key="k") + + +# ── The pending file under an agent ────────────────────────────── + + +def plant_pending(env, mode=0o600): + run(env) + path = pending_path(env) + os.chmod(path, mode) + return path, path.read_bytes() + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +def test_writable_pending_file_is_ignored_and_kept(env, capsys): + path, before = plant_pending(env, 0o666) + capsys.readouterr() + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert path.read_bytes() == before + assert stat.S_IMODE(path.stat().st_mode) == 0o666 + warns = [ + line + for line in capsys.readouterr().err.splitlines() + if line.startswith("WARN:") + ] + assert warns == [ + "WARN: Ignored the pending login in ~/.deepctl/login-pending.json: in an AI " + "agent session, dg doesn't use a file that can be changed by other users." + ] + assert [r.split()[0] for r in env["fake"].requests] == ["code"] + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX owners") +def test_foreign_pending_file_is_ignored(env, monkeypatch, capsys): + path, before = plant_pending(env) + other = os.geteuid() + 1 + monkeypatch.setattr(login_state.os, "geteuid", lambda: other) + # The pre-flight's own owner checks use the real uid. + monkeypatch.setattr(login_module, "_preflight", lambda config: None) + capsys.readouterr() + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert path.read_bytes() == before + assert "is owned by another user" in capsys.readouterr().err + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinks") +def test_symlinked_pending_file_is_ignored(env, tmp_path, capsys): + run(env) + path = pending_path(env) + elsewhere = tmp_path / "planted.json" + elsewhere.write_bytes(path.read_bytes()) + path.unlink() + path.symlink_to(elsewhere) + capsys.readouterr() + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert path.is_symlink() and elsewhere.exists() + assert "WARN:" in capsys.readouterr().err + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +def test_writable_pending_file_is_used_without_an_agent(env, agent, capsys): + path, _ = plant_pending(env, 0o666) + agent(False) + _, result = run(env, complete=True) + assert result.code == "auth_pending" + assert "WARN:" not in capsys.readouterr().err + + +def test_pending_file_never_holds_the_key(env): + write_user(env) + run(env) + env["fake"].answers = [DevicePoll("pending")] + run(env, complete=True) + assert KEY not in pending_path(env).read_text() + + +# ── Unwritable state ───────────────────────────────────────────── + + +@pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="root ignores modes" +) +@pytest.mark.parametrize( + "kwargs", [{}, {"api_key": "k", "yes": True}, {"complete": True}] +) +def test_unwritable_home_stops_before_any_request(env, monkeypatch, kwargs, capsys): + verified = [] + monkeypatch.setattr( + AuthManager, "verify_credentials", lambda self, *a, **k: verified.append(1) + ) + os.chmod(env["home"], 0o500) + try: + cmd, result = run(env, **kwargs) + finally: + os.chmod(env["home"], 0o700) + assert isinstance(result, LoginCodedError) + assert result.code == "state_unwritable" + assert result.next_step is None + assert result.details == {"dir": str(env["home"] / ".deepctl")} + assert result.hint == ( + "This environment can't store credentials. Ask the user to run dg login " + "in their own terminal." + ) + assert env["fake"].requests == [] and verified == [] + + +def test_save_failure_after_the_request_is_state_unwritable(env, monkeypatch): + def refuse(pending): + raise PermissionError(13, "denied") + + monkeypatch.setattr(login_module, "save_pending", refuse) + _, result = run(env) + assert result.code == "state_unwritable" + + +# ── An untrusted user config under an agent ────────────────────── + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +@pytest.mark.parametrize( + "kwargs", [{}, {"api_key": "k", "yes": True}, {"complete": True}] +) +@pytest.mark.parametrize("case", ["writable", "foreign"]) +def test_untrusted_config_stops_before_any_request( + env, monkeypatch, kwargs, case, capsys +): + if kwargs.get("complete"): + write_user(env) + run(env) # a valid pending login + env["fake"].requests.clear() + write_user(env, mode=0o666 if case == "writable" else 0o600) + if case == "foreign": + from deepctl_core import config as config_module + + other = os.getuid() + 1 + monkeypatch.setattr(config_module.os, "geteuid", lambda: other) + before = env["user"].read_bytes() + verified = [] + monkeypatch.setattr( + AuthManager, "verify_credentials", lambda self, *a, **k: verified.append(1) + ) + capsys.readouterr() + cmd, result = run(env, **kwargs) + why = ( + "can be changed by other users" + if case == "writable" + else "is owned by another user" + ) + assert dumped(result) == { + "error": ( + "In an AI agent session, dg can't save a login to " + "~/.config/deepctl/config.yaml " + f"because the file {why}. Nothing was sent or saved." + ), + "status": "error", + "code": "state_unwritable", + "hint": "A person must fix this file. If you are an AI agent, ask the user.", + "next_step": None, + "schema_version": 1, + "details": {"reason": "untrusted_config", "path": str(env["user"])}, + } + assert cmd.exit_code_for(result) == 1 + err = capsys.readouterr().err + assert "HINT: A person must fix this file." in err + assert env["user"].read_bytes() == before + if case == "writable": + assert stat.S_IMODE(env["user"].stat().st_mode) == 0o666 + assert env["fake"].requests == [] and verified == [] + assert env["keyring"] == {} + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +def test_untrusted_config_comes_before_the_env_key_trap(env, monkeypatch): + write_user(env, mode=0o666) + monkeypatch.setenv("DEEPGRAM_API_KEY", "envkey") + _, result = run(env) + assert result.code == "state_unwritable" + assert result.details["reason"] == "untrusted_config" + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +def test_writable_config_without_an_agent_still_logs_in(env, agent): + agent(False) + write_user(env, mode=0o666) + _, result = run(env, no_browser=True) + assert isinstance(result, LoginPendingResult) + + +# ── mode A stderr ──────────────────────────────────────────────── + + +def test_mode_a_stderr_keeps_only_prefixed_lines(): + import io + + real = io.StringIO() + filtered = login_module._ModeAStderr(real) + for text in ( + "Starting web authentication...\n", + "INFO: link\n", + "✓ API key stored\n", + "Warning: Project ID not provided\n", + "Error: something only chrome says\n", + "HINT: h\n", + ): + filtered.write(text) + assert real.getvalue() == "INFO: link\nHINT: h\n" + + +def test_mode_a_stderr_verbose_keeps_chrome_as_info_lines(): + import io + + real = io.StringIO() + filtered = login_module._ModeAStderr(real, verbose=True) + filtered.write("Starting web authentication...\n") + filtered.write("\nYou are now\nlogged in\n") + filtered.write("WARN: w\n") + assert real.getvalue() == ( + "INFO: Starting web authentication...\nINFO: You are now\nINFO: logged in\n" + "WARN: w\n" + ) + + +def test_api_key_login_in_agent_mode_has_empty_stderr(env, monkeypatch, capsys): + """J4.3: no project ID is chrome, not a warning, in mode A.""" + monkeypatch.setattr( + AuthManager, "verify_credentials", lambda self, *a, **k: (True, "ok", None) + ) + write_user(env) + capsys.readouterr() + _, result = run(env, api_key="0123456789abcdef0123456789abcdef01234567") + assert result.status == "success" + assert capsys.readouterr().err == "" + + +# ── Edge cases ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize("server_interval", [1, 0, 5]) +def test_poll_interval_has_a_6_s_floor(env, server_interval): + env["fake"].interval = server_interval + run(env) + assert json.loads(pending_path(env).read_text())["interval"] == 6 + _, result = run(env, complete=True) + assert result.details["retry_after"] >= 6 + + +def test_slow_down_honours_retry_after(env): + run(env) + env["fake"].answers = [DevicePoll("slow_down", retry_after=30)] + _, result = run(env, complete=True) + assert result.details["retry_after"] == 30 + assert json.loads(pending_path(env).read_text())["interval"] == 30 + + +def test_slow_down_adds_5_s(env): + run(env) + env["fake"].answers = [DevicePoll("slow_down")] + _, result = run(env, complete=True) + assert result.details["retry_after"] == 11 + + +def test_wait_backs_off_on_server_errors(env, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(login_module.time, "sleep", slept.append) + run(env) + env["fake"].answers = [ + DevicePoll("unavailable", detail="HTTP 503", retry_after=60), + DevicePoll("unavailable", detail="HTTP 502"), + approved(), + ] + write_user(env) + _, result = run(env, complete=True, wait=True) + assert result.status == "success" + assert slept[0] == 60 and slept[1] == 65 + + +def test_server_error_without_wait_keeps_the_pending_login(env): + run(env) + env["fake"].answers = [DevicePoll("unavailable", detail="HTTP 503")] + _, result = run(env, complete=True) + assert result.status == "error" + assert pending_path(env).exists() + + +def test_wait_stops_at_local_expiry(env, monkeypatch): + clock = {"now": time.time()} + monkeypatch.setattr( + login_module.time, "sleep", lambda s: clock.__setitem__("now", clock["now"] + s) + ) + monkeypatch.setattr(login_state.time, "time", lambda: clock["now"]) + monkeypatch.setattr(login_module.time, "time", lambda: clock["now"]) + run(env) + env["fake"].answers = [DevicePoll("unavailable", retry_after=3600)] * 3 + _, result = run(env, complete=True, wait=True) + assert result.code == "auth_expired" + + +@pytest.mark.parametrize("body", [[], None, "text", 7]) +def test_unreadable_device_code_answer_is_coded(env, monkeypatch, capsys, body): + from unittest.mock import Mock + + def device_code(self, identity_url=None): + response = Mock(status_code=201) + response.json.return_value = body + self.client.post = Mock(return_value=response) + return _REAL_REQUEST(self, identity_url) + + monkeypatch.setattr(AuthManager, "_request_device_code", device_code) + for kwargs in ({}, {"wait": True}): + capsys.readouterr() + cmd, result = run(env, **kwargs) + assert dumped(result) == { + "error": "Got a response from id.dx.deepgram.com that dg couldn't read.", + "status": "error", + "code": "network_unavailable", + "hint": ( + "Retry once. If it fails again, tell the user that something on this " + "network (a proxy, captive portal or firewall) is answering in " + "Deepgram's place." + ), + "next_step": None, + "schema_version": 1, + "details": {"host": "id.dx.deepgram.com", "reason": "unreadable response"}, + } + assert cmd.exit_code_for(result) == 1 + assert "TypeError" not in capsys.readouterr().err + assert not pending_path(env).exists() + + +@pytest.mark.parametrize("interval", [-3, "6", None, True, float("nan")]) +def test_unusable_pending_interval_is_no_pending_login(env, interval): + run(env) + data = json.loads(pending_path(env).read_text()) + data["interval"] = interval + pending_path(env).write_text(json.dumps(data)) + os.chmod(pending_path(env), 0o600) + _, result = run(env, complete=True) + assert result.code == "auth_expired" + assert [r.split()[0] for r in env["fake"].requests] == ["code"] + + +def test_small_pending_interval_is_raised_to_the_floor(env): + run(env) + data = json.loads(pending_path(env).read_text()) + data["interval"] = 0 + pending_path(env).write_text(json.dumps(data)) + os.chmod(pending_path(env), 0o600) + _, result = run(env, complete=True) + assert result.details["retry_after"] == 6 + + +def test_complete_with_another_profile_is_a_usage_error(env): + import click + + run(env, profile="work") + with pytest.raises(click.UsageError, match='pending login is for profile "work"'): + run(env, complete=True, profile="other") + assert [r.split()[0] for r in env["fake"].requests] == ["code"] + env["fake"].answers = [DevicePoll("pending")] + _, result = run(env, complete=True, profile="work") + assert result.code == "auth_pending" + + +def test_complete_with_save_base_url_is_a_usage_error(env): + import click + + with pytest.raises(click.UsageError): + run(env, complete=True, save_base_url=True) diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_strings.py b/packages/deepctl-cmd-login/tests/unit/test_login_strings.py new file mode 100644 index 00000000..a5550789 --- /dev/null +++ b/packages/deepctl-cmd-login/tests/unit/test_login_strings.py @@ -0,0 +1,79 @@ +"""Login's user-facing strings equal the error catalogue's. + +Runs when DG_ERROR_CODES_JSON names the catalogue file; skipped otherwise. +""" + +from __future__ import annotations + +import json +import os + +import pytest +from deepctl_core import messages as msg + +CATALOGUE = os.environ.get("DG_ERROR_CODES_JSON") +pytestmark = pytest.mark.skipif(not CATALOGUE, reason="DG_ERROR_CODES_JSON not set") + +# catalogue id -> messages.py constant +IDS = { + "E-CONFIRM-ENVKEY": "E_CONFIRM_ENVKEY", + "A-CONFIRM-ENVKEY": "A_CONFIRM_ENVKEY", + "A-CONFIRM-ENVKEY-NULL": "A_CONFIRM_ENVKEY_NULL", + "C-CONFIRM-ENVKEY-NULL": "C_CONFIRM_ENVKEY_NULL", + "E-CONFIRM-RELOGIN": "E_CONFIRM_RELOGIN", + "H-CONFIRM-YES": "H_CONFIRM_YES", + "A-CONFIRM-YES": "A_CONFIRM_YES", + "A-CONFIRM-YES-NULL": "A_CONFIRM_YES_NULL", + "C-CONFIRM-YES-NULL": "C_CONFIRM_YES_NULL", + "M-LOGIN-PENDING": "M_LOGIN_PENDING", + "I-LOGIN-PENDING": "I_LOGIN_PENDING", + "A-LOGIN-PENDING": "A_LOGIN_PENDING", + "H-LOGIN-PENDING": "H_LOGIN_PENDING", + "E-AUTH-PENDING": "E_AUTH_PENDING", + "H-AUTH-PENDING": "H_AUTH_PENDING", + "A-AUTH-PENDING": "A_AUTH_PENDING", + "E-AUTH-EXPIRED": "E_AUTH_EXPIRED", + "H-AUTH-EXPIRED": "H_AUTH_EXPIRED", + "A-AUTH-EXPIRED": "A_AUTH_EXPIRED", + "E-AUTH-DENIED": "E_AUTH_DENIED", + "H-AUTH-DENIED": "H_AUTH_DENIED", + "A-AUTH-DENIED": "A_AUTH_DENIED", + "E-STATE-UNWRITABLE": "E_STATE_UNWRITABLE", + "H-STATE-UNWRITABLE": "H_STATE_UNWRITABLE", + "A-STATE-UNWRITABLE": "A_STATE_UNWRITABLE", + "E-CFG-SAVE-UNTRUSTED": "E_CFG_SAVE_UNTRUSTED", + "W-LOGIN-PENDING-UNTRUSTED": "W_LOGIN_PENDING_UNTRUSTED", + "E-NETWORK-UNREADABLE": "E_NETWORK_UNREADABLE", + "H-NETWORK-UNREADABLE": "H_NETWORK_UNREADABLE", + "A-NETWORK-UNREADABLE": "A_NETWORK_UNREADABLE", +} + + +@pytest.fixture(scope="module") +def catalogue(): + with open(CATALOGUE, encoding="utf-8") as f: + return json.load(f) + + +@pytest.mark.parametrize(("cid", "const"), sorted(IDS.items())) +def test_string_matches_the_catalogue(catalogue, cid, const): + assert getattr(msg, const) == catalogue["strings"][cid] + + +def test_codes_and_details_keys(catalogue): + codes = catalogue["codes"] + used = { + msg.CODE_CONFIRMATION_REQUIRED: {"flag"}, + msg.CODE_AUTH_PENDING: {"retry_after", "verification_uri", "user_code"}, + msg.CODE_AUTH_EXPIRED: set(), + msg.CODE_AUTH_DENIED: set(), + msg.CODE_STATE_UNWRITABLE: {"dir", "reason", "path"}, + msg.CODE_NETWORK_UNAVAILABLE: {"host", "reason"}, + } + for code, keys in used.items(): + assert code in codes + assert keys <= set(codes[code].get("details") or []) + reasons = codes["state_unwritable"]["details_values"]["E-CFG-SAVE-UNTRUSTED"] + assert reasons["reason"] == ["untrusted_config"] + unreadable = codes["network_unavailable"]["details_values"]["E-NETWORK-UNREADABLE"] + assert unreadable["reason"] == ["unreadable response"] diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index 42ae5da3..5261ebcf 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -3,17 +3,19 @@ import os import time import webbrowser +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any -from urllib.parse import urlencode +from urllib.parse import urlsplit -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from . import hosts, output from . import messages as msg from .client import _split_base_url from .config import Config from .hosts import check_credential_host, check_identity_host +from .login_state import MIN_POLL_INTERVAL from .models import ProfileInfo, ProfilesResult from .output import stderr_console @@ -66,6 +68,21 @@ def _load_modules() -> None: # Keyring service identifier using reverse domain notation KEYRING_SERVICE = "com.deepgram.dx.deepctl" +# Sent where the identity provider asks for a device name: a fixed value, +# never this machine's hostname. +DEVICE_NAME = "deepctl" +DEVICE_CODE_GRANT = "urn:ietf:params:oauth:grant-type:device_code" + + +def _https_or_loopback(url: str | None) -> str | None: + """``url`` when it is https, or http to a loopback host; else ValueError.""" + if url is None: + return None + origin = hosts.parse_origin(url) + if origin is None or not (origin.secure or hosts.is_loopback(origin.host)): + raise ValueError("the login link must use https") + return url + class DeviceCodeResponse(BaseModel): """Response from device code request.""" @@ -75,18 +92,47 @@ class DeviceCodeResponse(BaseModel): verification_uri: str verification_uri_complete: str | None = None expires_in: int - interval: int + interval: int = 5 + + _check_uris = field_validator("verification_uri", "verification_uri_complete")( + _https_or_loopback + ) class TokenResponse(BaseModel): - """Response from token request.""" + """Response from token request (``pid`` is the RFC 8628 path's name for + the project ID).""" access_token: str - project_id: str + project_id: str | None = None + pid: str | None = None dg_token: str | None = None dg_expires_in: int | None = None +@dataclass(frozen=True) +class DevicePoll: + """One answer from the token endpoint. + + ``state`` is ``token`` (``token`` is set), ``pending``, ``slow_down``, + ``denied``, ``expired`` or ``error`` (``detail`` says why). + """ + + state: str + token: TokenResponse | None = None + detail: str = "" + retry_after: int | None = None + + +def _retry_after(response: "httpx.Response") -> int | None: + """The server's Retry-After in whole seconds (delta form only), or None.""" + value = response.headers.get("Retry-After") + if not isinstance(value, str) or not value.strip().isdigit(): + return None + value = value.strip() + return min(int(value), 3600) + + def _insecure_allowed() -> bool: """Whether DEEPGRAM_CLI_INSECURE may turn off TLS verification. @@ -137,12 +183,28 @@ def _warn_if_verified(request: "httpx.Request") -> None: output.print_hint(escape(msg.W_INSECURE_REMOTE_HINT)) +def escape(text: str) -> str: + """rich.markup.escape, imported on first use (no rich at import).""" + from rich.markup import escape as rich_escape + + return rich_escape(text) + + class AuthenticationError(Exception): """Authentication related errors.""" pass +class DeviceCodeUnreadable(AuthenticationError): + """The device-code answer from ``host`` couldn't be read (not a JSON + object with the expected fields): ``network_unavailable``.""" + + def __init__(self, host: str) -> None: + super().__init__(f"unreadable device code response from {host}") + self.host = host + + class AuthManager: """Cross-platform authentication manager.""" @@ -587,13 +649,12 @@ def login_with_api_key( elif error_type == "project": # Project ID is invalid, but the key might be fine — warn but # still store the key so the user isn't locked out. - console.print( - f"[yellow]Warning:[/yellow] {message}. " - "API key will still be stored." + output.print_warning( + escape(f"{message}. API key will still be stored."), stderr=True ) console.print( - "[dim]You can update the project ID later with: " - "deepctl login --project-id [/dim]" + "[dim]You can set the project ID later with: " + f"{transport.prog() or 'dg'} login --project-id [/dim]" ) # Clear the bad project_id so we don't store it project_id = None @@ -614,8 +675,13 @@ def login_with_api_key( console.print("[green]✓[/green] API key stored securely in system keyring") keyring_available = True except Exception as e: - console.print(f"[yellow]Warning:[/yellow] Could not store in keyring: {e}") - console.print("API key will be stored in config file instead") + output.print_warning( + escape( + f"Could not store in keyring: {e}. " + "The API key will be stored in the config file instead." + ), + stderr=True, + ) # Update config with non-sensitive data # Only store API key in config if keyring is not available @@ -631,7 +697,7 @@ def login_with_api_key( if project_id: console.print(f"[dim]Project ID:[/dim] {project_id}") - def login_with_device_flow(self) -> None: + def login_with_device_flow(self, open_browser: bool = True) -> None: """Login using device flow (interactive method).""" console.print("[blue]Starting device flow authentication...[/blue]") @@ -639,9 +705,6 @@ def login_with_device_flow(self) -> None: identity_url = check_identity_host(self.config) try: - # Get hostname for device identification - hostname = os.uname().nodename if hasattr(os, "uname") else "unknown" - # Request device code from dx-id auth provider device_response = self._request_device_code(identity_url) @@ -651,28 +714,37 @@ def login_with_device_flow(self) -> None: ) # Open browser immediately and start polling - console.print( - "\n[bold]Hello from Deepgram![/bold] Opening browser to " - "complete login..." + if open_browser: + console.print( + "\n[bold]Hello from Deepgram![/bold] Opening browser to " + "complete login..." + ) + else: + console.print( + "\n[bold]Hello from Deepgram![/bold] Open the link below to " + "complete login." + ) + output.print_info( + escape( + msg.I_LOGIN_PENDING.format( + verification_uri=msg.display(device_response.verification_uri), + user_code=msg.display(device_response.user_code or ""), + ) + ), + stderr=True, ) - if device_response.user_code: - console.print(f"[bold]Enter code:[/bold] {device_response.user_code}") - - try: - webbrowser.open(verification_uri) - console.print("[green]✓[/green] Opened browser for authentication") - except Exception as e: - console.print(f"[yellow]Warning:[/yellow] Could not open browser: {e}") - console.print( - f"[dim]If the browser didn't open, visit:[/dim] " - f"[dim]{verification_uri}[/dim]\n" - ) + if open_browser: + try: + webbrowser.open(verification_uri) + console.print("[green]✓[/green] Opened browser for authentication") + except Exception as e: + console.print( + f"[yellow]Warning:[/yellow] Could not open browser: {e}" + ) # Poll for token (starts immediately — user may have clicked the link) - token_response = self._poll_for_token( - device_response, hostname, identity_url - ) + token_response = self._poll_for_token(device_response, identity_url) # Store token and get user info self._store_token(token_response) @@ -680,6 +752,8 @@ def login_with_device_flow(self) -> None: console.print("\n[green]Key created and stored successfully.[/green]") console.print("\nYou are now logged in. Happy coding!") + except DeviceCodeUnreadable: + raise except Exception as e: console.print(f"[red]Error during device flow:[/red] {e}") raise AuthenticationError(f"Device flow failed: {e}") @@ -692,11 +766,9 @@ def _request_device_code( Returns: DeviceCodeResponse with device code and verification URI """ - hostname = os.uname().nodename if hasattr(os, "uname") else "unknown" - payload = { "client_id": CLIENT_ID, - "hostname": hostname, + "hostname": DEVICE_NAME, "scopes": "openid profile email", } @@ -708,8 +780,18 @@ def _request_device_code( headers={"Content-Type": "application/json"}, ) - if response.status_code == 201: - return DeviceCodeResponse(**response.json()) + if response.status_code in (200, 201): + host = urlsplit(base).hostname or base + try: + body = response.json() + except ValueError: + raise DeviceCodeUnreadable(host) from None + if not isinstance(body, dict): + raise DeviceCodeUnreadable(host) + try: + return DeviceCodeResponse(**body) + except (ValueError, TypeError): # incl. pydantic ValidationError + raise DeviceCodeUnreadable(host) from None else: raise AuthenticationError( f"Device code request failed: {response.status_code}" @@ -718,25 +800,69 @@ def _request_device_code( except httpx.RequestError as e: raise AuthenticationError(f"Network error during device code request: {e}") + def poll_device_token( + self, device_code: str, identity_url: str | None = None + ) -> DevicePoll: + """Ask the token endpoint once whether ``device_code`` was approved. + + A POST (RFC 8628 §3.4), so the device code never lands in a URL, + and no device name is sent. Network errors are raised + (``httpx.RequestError``). + """ + base = identity_url or check_identity_host(self.config) + response = self.client.post( + f"{base}/token", + data={ + "grant_type": DEVICE_CODE_GRANT, + "device_code": device_code, + "client_id": CLIENT_ID, + }, + ) + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict): + body = {} + if response.status_code in (200, 201): + try: + return DevicePoll("token", token=TokenResponse(**body)) + except ValueError: + return DevicePoll("error", detail="unusable token response") + if response.status_code == 429: + return DevicePoll("slow_down", retry_after=_retry_after(response)) + error = body.get("error") + if response.status_code >= 500 or error == "temporarily_unavailable": + # A transient server problem: back off and ask again. + return DevicePoll( + "unavailable", + detail=f"HTTP {response.status_code}", + retry_after=_retry_after(response), + ) + state = { + "authorization_pending": "pending", + "slow_down": "slow_down", + "access_denied": "denied", + "expired_token": "expired", + }.get(error if isinstance(error, str) else "") + if state is not None: + return DevicePoll(state) + reason = error if isinstance(error, str) else "unknown error" + return DevicePoll( + "error", detail=f"HTTP {response.status_code}: {msg.display(reason)[:80]}" + ) + def _poll_for_token( self, device_response: DeviceCodeResponse, - hostname: str, identity_url: str | None = None, ) -> TokenResponse: - """Poll for token using device code.""" + """Poll for token using device code until approved, denied or expired.""" console.print("\n[blue]Waiting for authentication...[/blue]") start_time = time.time() - - query_params = { - "device_code": device_response.device_code, - "client_id": CLIENT_ID, - "hostname": hostname, - } - + interval = max(MIN_POLL_INTERVAL, device_response.interval) base = identity_url or check_identity_host(self.config) - poll_url = f"{base}/device/token?{urlencode(query_params)}" from rich.progress import Progress, SpinnerColumn, TextColumn @@ -750,28 +876,26 @@ def _poll_for_token( while time.time() - start_time < device_response.expires_in: try: - # Use GET request like Go implementation - response = self.client.get(poll_url) - - if response.status_code == 201: - response_data = response.json() - return TokenResponse(**response_data) - elif response.status_code == 404: - # Still pending - this is the expected status code from - # Go implementation - time.sleep(device_response.interval) - continue - else: - error_data = response.json() - raise AuthenticationError( - f"Token request failed: " - f"{error_data.get('error', 'Unknown error')}" - ) - + poll = self.poll_device_token(device_response.device_code, base) except httpx.RequestError as e: console.print(f"[red]Network error:[/red] {e}") - time.sleep(device_response.interval) + time.sleep(interval) + continue + if poll.state == "token" and poll.token is not None: + return poll.token + if poll.state == "pending": + time.sleep(interval) continue + if poll.state in ("slow_down", "unavailable"): + interval = max(interval + 5, poll.retry_after or 0) + left = device_response.expires_in - (time.time() - start_time) + time.sleep(max(0.0, min(interval, left))) + continue + if poll.state == "denied": + raise AuthenticationError(msg.E_AUTH_DENIED) + if poll.state == "expired": + break + raise AuthenticationError(str(poll.detail)) raise AuthenticationError("Authentication timed out") @@ -804,10 +928,13 @@ def _store_token(self, token_response: TokenResponse) -> None: ) keyring_available = True except Exception as e: - console.print( - f"[yellow]Warning:[/yellow] Could not store in keyring: {e}" + output.print_warning( + escape( + f"Could not store in keyring: {e}. " + "The API key will be stored in the config file instead." + ), + stderr=True, ) - console.print("API key will be stored in config file instead") for keyring_key in (f"jwt.{profile_name}", f"refresh-token.{profile_name}"): self._delete_keyring_credential(keyring_key) @@ -815,7 +942,7 @@ def _store_token(self, token_response: TokenResponse) -> None: self.config.create_profile( profile_name, api_key=api_key if not keyring_available else None, - project_id=token_response.project_id, + project_id=token_response.project_id or token_response.pid, ) profile = self.config.get_profile(profile_name) profile.dg_token_expires_at = expires_at @@ -824,10 +951,6 @@ def _store_token(self, token_response: TokenResponse) -> None: console.print( "[green]✓[/green] API key stored securely in system keyring" ) - else: - console.print( - "[yellow]Warning:[/yellow] API key stored in the config file fallback" - ) return raise AuthenticationError( diff --git a/packages/deepctl-core/src/deepctl_core/config.py b/packages/deepctl-core/src/deepctl_core/config.py index 9befabfe..b4d12e6f 100644 --- a/packages/deepctl-core/src/deepctl_core/config.py +++ b/packages/deepctl-core/src/deepctl_core/config.py @@ -1000,17 +1000,20 @@ def _save_refusal(st: os.stat_result | None) -> str | None: class ConfigSaveRefused(OSError): """An explicit save refused in an AI agent session; ``hint`` says who - fixes it.""" + fixes it, ``why`` is the cause (a W_CFG_UNTRUSTED_WHY key).""" hint = msg.W_CFG_UNTRUSTED_FILE_HINT + why: str = "" def _save_refused(why: str, config_path: Path) -> ConfigSaveRefused: - return ConfigSaveRefused( + exc = ConfigSaveRefused( errno.EPERM, msg.W_CFG_SAVE_UNTRUSTED.format(why=msg.W_CFG_UNTRUSTED_WHY[why]), str(config_path), ) + exc.why = why + return exc @contextmanager @@ -1308,7 +1311,10 @@ def _get_default_config_path(self) -> Path: """Get the default configuration path for the current platform.""" # Use platformdirs for cross-platform config directory config_dir = Path(platformdirs.user_config_dir("deepctl", "deepgram")) - _private_mkdir(config_dir) + # A home dg can't write (a read-only mount) still runs every command; + # a save reports the error, and dg login stops with state_unwritable. + with suppress(OSError): + _private_mkdir(config_dir) # Migrate from old location if needed self._migrate_config_if_needed(config_dir) @@ -1332,14 +1338,17 @@ def _migrate_config_if_needed(self, new_config_dir: Path) -> None: # If old config exists and new doesn't, migrate it if old_config_path.exists() and not new_config_path.exists(): try: - print(f"Migrating config from {old_config_path} to {new_config_path}") + print( + f"Migrating config from {old_config_path} to {new_config_path}", + file=sys.stderr, + ) text = old_config_path.read_text(encoding="utf-8") # Written like any save: 0600, atomic, never via the source's mode. with _directory_lock(new_config_dir): self._replace_or_rewrite(new_config_path, text) - print("✓ Config migrated successfully") + print("✓ Config migrated successfully", file=sys.stderr) except Exception as e: - print(f"Warning: Could not migrate config: {e}") + print(f"Warning: Could not migrate config: {e}", file=sys.stderr) def _get_project_config_path(self) -> Path: """Get the project-specific configuration path.""" diff --git a/packages/deepctl-core/src/deepctl_core/login_state.py b/packages/deepctl-core/src/deepctl_core/login_state.py new file mode 100644 index 00000000..2a5eeb58 --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/login_state.py @@ -0,0 +1,307 @@ +"""dg's state directory and the pending device-login store. + +``state_dir()`` is the one helper every state file goes through: +``~/.deepctl`` in 0.5.0; a later release repoints it. + +``dg login`` in an agent session asks for a device code, saves it here and +returns at once; ``dg login --complete`` polls once with it. The file holds +the device code, its user code, link, poll interval, expiry, the identity +host and the profile. Never a key. +""" + +from __future__ import annotations + +import dataclasses +import errno +import json +import math +import os +import secrets +import stat +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +PENDING_NAME = "login-pending.json" +# The shortest poll interval dg uses, whatever the server says: the server allows +# 60 token requests per 5 minutes per address, and 6 s leaves headroom. +MIN_POLL_INTERVAL = 6 +_MAX_PENDING_BYTES = 8192 +_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) +_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0) +_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0) +_O_BINARY = getattr(os, "O_BINARY", 0) + +# Why a pending file fails the agent-session trust test; the +# values are msg.W_CFG_UNTRUSTED_WHY keys. +WHY_SYMLINK = "symlink" +WHY_OWNER = "owner" +WHY_WRITABLE = "writable" +WHY_NOT_REGULAR = "not_regular" +WHY_LINKS = "links" + + +def state_dir() -> Path: + """dg's state directory (``~/.deepctl`` in 0.5.0).""" + return Path.home() / ".deepctl" + + +class StateDirError(OSError): + """The state directory can't hold a pending login (``state_unwritable``).""" + + +def unwritable_dir(directory: Path, file: Path | None = None) -> Path | None: + """``directory`` when dg couldn't create or write a file in it, else None. + + Checks without creating anything: the nearest existing ancestor must be + a directory this process may write and search. ``file`` (an existing + file in ``directory`` that dg would rewrite in place) also counts as + writable. + """ + if file is not None: + try: + if os.path.isfile(file) and os.access(file, os.W_OK): + return None + except OSError: + pass + probe = directory + while True: + try: + st = os.stat(probe) + except FileNotFoundError: + parent = probe.parent + if parent == probe: + return directory + probe = parent + continue + except OSError: + return directory + if not stat.S_ISDIR(st.st_mode): + return directory + return None if os.access(probe, os.W_OK | os.X_OK) else directory + + +@dataclass(frozen=True) +class PendingLogin: + """A device code waiting for approval.""" + + device_code: str + user_code: str + verification_uri: str + interval: int + expires_at: float + host: str + profile: str + + def expired(self, now: float | None = None) -> bool: + return (time.time() if now is None else now) >= self.expires_at + + def expires_in(self, now: float | None = None) -> int: + left = self.expires_at - (time.time() if now is None else now) + return max(0, int(left)) + + +def _from_json(data: object) -> PendingLogin | None: + if not isinstance(data, dict): + return None + try: + pending = PendingLogin( + device_code=data["device_code"], + user_code=data["user_code"], + verification_uri=data["verification_uri"], + interval=data["interval"], + expires_at=data["expires_at"], + host=data["host"], + profile=data["profile"], + ) + except (KeyError, TypeError): + return None + strings = ( + pending.device_code, + pending.user_code, + pending.verification_uri, + pending.host, + pending.profile, + ) + if not all(isinstance(value, str) and value for value in strings): + return None + interval = pending.interval + if ( + not isinstance(interval, (int, float)) + or isinstance(interval, bool) + or interval != interval # NaN + or interval < 0 + or interval == float("inf") + ): + return None # not a usable interval: no pending login + pending = dataclasses.replace( + pending, interval=max(MIN_POLL_INTERVAL, math.ceil(interval)) + ) + if not isinstance(pending.expires_at, (int, float)) or isinstance( + pending.expires_at, bool + ): + return None + return pending + + +def _open_dir(directory: Path, *, create: bool) -> int: + """Open ``directory`` (made 0700 when ``create``) and check it. + + It must be a directory owned by this user; one the user can't keep + private is refused. Group or other bits on the user's own directory are + removed, so the pending file is in a 0700 directory. + """ + if create: + try: + os.makedirs(directory.parent, exist_ok=True) + os.mkdir(directory, 0o700) + except FileExistsError: + pass + fd = os.open(directory, os.O_RDONLY | _O_DIRECTORY) + try: + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode): + raise StateDirError(errno.ENOTDIR, "not a directory", str(directory)) + if sys.platform != "win32": + if st.st_uid != os.geteuid(): + raise StateDirError( + errno.EPERM, "owned by another user", str(directory) + ) + if create and stat.S_IMODE(st.st_mode) & 0o077: + os.fchmod(fd, 0o700) + except BaseException: + os.close(fd) + raise + return fd + + +def save_pending(pending: PendingLogin) -> Path: + """Write ``pending`` to ``{state_dir}/login-pending.json`` at 0600. + + The state directory is created 0700. The new content goes to a fresh + 0600 file (``O_CREAT|O_EXCL|O_NOFOLLOW``) next to the target and + replaces it atomically, so a reader never sees half a file and a + planted symlink is never followed. Raises OSError (StateDirError for a + directory dg can't use). + """ + directory = state_dir() + target = directory / PENDING_NAME + data = (json.dumps(asdict(pending), indent=2) + "\n").encode("utf-8") + tmp_name = f".{PENDING_NAME}.{secrets.token_hex(8)}.tmp" + dir_fd = _open_dir(directory, create=True) + use_dir_fd = sys.platform != "win32" and os.open in os.supports_dir_fd + try: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW | _O_BINARY + if use_dir_fd: + fd = os.open(tmp_name, flags, 0o600, dir_fd=dir_fd) + else: + fd = os.open(directory / tmp_name, flags, 0o600) + try: + try: + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + os.fsync(fd) + finally: + os.close(fd) + if use_dir_fd: + os.replace(tmp_name, PENDING_NAME, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) + else: + os.replace(directory / tmp_name, target) + except BaseException: + try: + if use_dir_fd: + os.unlink(tmp_name, dir_fd=dir_fd) + else: + os.unlink(directory / tmp_name) + except OSError: + pass + raise + finally: + os.close(dir_fd) + return target + + +@dataclass(frozen=True) +class LoadedPending: + """What ``load_pending`` found: the login, or why the file was ignored.""" + + pending: PendingLogin | None + path: Path + ignored_why: str | None = None + identity: tuple[int, int] | None = None + + +def load_pending(*, agent: bool) -> LoadedPending: + """Read the pending login, if there is a usable one. + + The file is opened without following a symlink and must be a regular + file. In an AI agent session (``agent``) it must also be owned by this + user, with one link and no group or other write bit; + otherwise it is ignored (``ignored_why`` says why) and left in place. + A missing or malformed file is simply absent. + """ + path = state_dir() / PENDING_NAME + try: + fd = os.open(path, os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK | _O_BINARY) + except FileNotFoundError: + return LoadedPending(None, path) + except OSError as exc: + if exc.errno in (errno.ELOOP, errno.EMLINK): + return LoadedPending(None, path, WHY_SYMLINK if agent else None) + if agent and sys.platform != "win32": + # e.g. EACCES: say so when the file belongs to someone else. + try: + if os.lstat(path).st_uid != os.geteuid(): + return LoadedPending(None, path, WHY_OWNER) + except OSError: + pass + return LoadedPending(None, path) + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): + return LoadedPending(None, path, WHY_NOT_REGULAR if agent else None) + if agent and sys.platform != "win32": + why = None + if st.st_uid != os.geteuid(): + why = WHY_OWNER + elif stat.S_IMODE(st.st_mode) & 0o022: + why = WHY_WRITABLE + elif st.st_nlink != 1: + why = WHY_LINKS + if why is not None: + return LoadedPending(None, path, why) + raw = os.read(fd, _MAX_PENDING_BYTES + 1) + finally: + os.close(fd) + identity = (st.st_dev, st.st_ino) + if len(raw) > _MAX_PENDING_BYTES: + return LoadedPending(None, path, identity=identity) + try: + data = json.loads(raw.decode("utf-8")) + except ValueError: + return LoadedPending(None, path, identity=identity) + return LoadedPending(_from_json(data), path, identity=identity) + + +def remove_pending(loaded: LoadedPending) -> None: + """Delete the pending file ``loaded`` came from, if it is still that file. + + A ``dg login`` replacing the file between the check and the unlink loses + its new code; the cost is one more ``dg login``. + """ + if loaded.identity is None: + return + try: + st = os.lstat(loaded.path) + except OSError: + return + if (st.st_dev, st.st_ino) != loaded.identity: + return # replaced by a newer dg login meanwhile: keep it + try: + os.unlink(loaded.path) + except FileNotFoundError: + pass diff --git a/packages/deepctl-core/src/deepctl_core/messages.py b/packages/deepctl-core/src/deepctl_core/messages.py index 18618905..870b51c6 100644 --- a/packages/deepctl-core/src/deepctl_core/messages.py +++ b/packages/deepctl-core/src/deepctl_core/messages.py @@ -162,6 +162,73 @@ C_CONFIRM_YES_NULL = ( "Add --yes to this command in the job if replacing the saved login is intended." ) + +# Device login for agents (catalogue strings). +CODE_AUTH_PENDING = "auth_pending" +CODE_AUTH_EXPIRED = "auth_expired" +CODE_AUTH_DENIED = "auth_denied" +CODE_STATE_UNWRITABLE = "state_unwritable" +M_LOGIN_PENDING = "Open the link and enter the code to finish logging in." +I_LOGIN_PENDING = ( + "To finish logging in, open {verification_uri} and enter the code {user_code}." +) +A_LOGIN_PENDING = ( + "Give the user the link and code, then run {prog} login --complete after " + "they approve." +) +# The person's variant of the line above (mode H/P). +H_LOGIN_PENDING = ( + "After you approve the login in your browser, run {prog} login --complete." +) +E_AUTH_PENDING = "Login is waiting for approval in the browser." +H_AUTH_PENDING = ( + "Open {verification_uri}, enter the code {user_code}, then run {prog} login " + "--complete." +) +A_AUTH_PENDING = ( + "Wait {retry_after} s for the user to approve the login at " + "{verification_uri} with the code {user_code}, then run {prog} login --complete." +) +E_AUTH_EXPIRED = "No login is waiting to be completed, or it expired." +H_AUTH_EXPIRED = "Run {prog} login to start again." +A_AUTH_EXPIRED = "Run {prog} login again and give the user the new link and code." +E_AUTH_DENIED = "The login request was denied in the browser." +H_AUTH_DENIED = "Run {prog} login to try again." +A_AUTH_DENIED = "The user denied the login. Ask them before trying again." +E_STATE_UNWRITABLE = "{prog} can't save a login here: {dir} isn't writable." +H_STATE_UNWRITABLE = ( + "Run {prog} login in a terminal where your home directory is writable." +) +A_STATE_UNWRITABLE = ( + "This environment can't store credentials. Ask the user to run {prog} login " + "in their own terminal." +) +# Login and an untrusted user config: the config file (``{path}``, shown with +# ``~``) can't be saved in this AI agent session, so login stops before +# anything is sent. Code state_unwritable; hint W_CFG_UNTRUSTED_FILE_HINT. +# {why} is a W_CFG_UNTRUSTED_WHY value. +E_CFG_SAVE_UNTRUSTED = ( + "In an AI agent session, {prog} can't save a login to {path} because the " + "file {why}. Nothing was sent or saved." +) +# A pending device-login file that fails the agent-session test; +# it is ignored and left in place. +W_LOGIN_PENDING_UNTRUSTED = ( + "Ignored the pending login in {path}: in an AI agent session, {prog} " + "doesn't use a file that {why}." +) +# The identity provider's device-code answer couldn't be read (code +# network_unavailable, details.reason "unreadable response"). +CODE_NETWORK_UNAVAILABLE = "network_unavailable" +E_NETWORK_UNREADABLE = "Got a response from {host} that {prog} couldn't read." +H_NETWORK_UNREADABLE = ( + "A proxy, captive portal or firewall may be answering in Deepgram's place. " + "Check that this network lets {prog} reach {host}, then try again." +) +A_NETWORK_UNREADABLE = ( + "Retry once. If it fails again, tell the user that something on this network " + "(a proxy, captive portal or firewall) is answering in Deepgram's place." +) E_SAVE_CUSTOM_CONFIG = ( "--save-base-url can't be used with --config: only the default user " "config ({user_config}) can approve API hosts. Run the command without " diff --git a/packages/deepctl-core/tests/unit/test_auth.py b/packages/deepctl-core/tests/unit/test_auth.py index 74959b68..6554b56b 100644 --- a/packages/deepctl-core/tests/unit/test_auth.py +++ b/packages/deepctl-core/tests/unit/test_auth.py @@ -1,8 +1,8 @@ """Tests for the authentication module.""" +import json from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, Mock, patch -from urllib.parse import parse_qs, urlparse import httpx import keyring.errors @@ -820,19 +820,21 @@ def test_get_api_key_uses_explicit_key(self, mock_config): assert auth.get_api_key() == "sk-explicit" - def test_poll_for_token_encodes_device_flow_parameters(self, mock_config): + def test_poll_for_token_posts_without_a_hostname(self, mock_config): + """RFC 8628 §3.4: the device code goes in a POST body, never a URL, and + no machine name is sent.""" from deepctl_core.auth import DeviceCodeResponse auth = self._make_auth(mock_config) response = Mock() - response.status_code = 201 + response.status_code = 200 response.json.return_value = { "access_token": "jwt", "dg_token": "dg-finite-key", "dg_expires_in": 3600, - "project_id": "project", + "pid": "project", } - auth.client.get.return_value = response + auth.client.post.return_value = response token = auth._poll_for_token( DeviceCodeResponse( @@ -841,16 +843,164 @@ def test_poll_for_token_encodes_device_flow_parameters(self, mock_config): expires_in=600, interval=1, ), - "test-host", + "https://id.example.test", ) - poll_url = auth.client.get.call_args.args[0] - assert parse_qs(urlparse(poll_url).query) == { - "device_code": ["device-code"], - "client_id": ["deepgram-cli"], - "hostname": ["test-host"], + auth.client.get.assert_not_called() + url = auth.client.post.call_args.args[0] + assert url == "https://id.example.test/token" + assert auth.client.post.call_args.kwargs["data"] == { + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": "device-code", + "client_id": "deepgram-cli", } assert token.dg_token == "dg-finite-key" + assert token.pid == "project" + + @pytest.mark.parametrize( + ("status", "body", "state"), + [ + (400, {"error": "authorization_pending"}, "pending"), + (400, {"error": "slow_down"}, "slow_down"), + (429, {}, "slow_down"), + (400, {"error": "access_denied"}, "denied"), + (400, {"error": "expired_token"}, "expired"), + (400, {"error": "invalid_grant"}, "error"), + (500, None, "unavailable"), + (503, {"error": "temporarily_unavailable"}, "unavailable"), + (400, {"error": "temporarily_unavailable"}, "unavailable"), + ], + ) + def test_poll_device_token_maps_rfc8628_answers( + self, mock_config, status, body, state + ): + auth = self._make_auth(mock_config) + response = Mock() + response.status_code = status + if body is None: + response.json.side_effect = ValueError("not json") + else: + response.json.return_value = body + auth.client.post.return_value = response + assert auth.poll_device_token("dc", "https://id.example.test").state == state + + @pytest.mark.parametrize(("status", "header", "want"), [(429, "30", 30), (503, "60", 60), (429, "Wed, 21 Oct 2026 07:28:00 GMT", None)]) + def test_poll_reads_retry_after(self, mock_config, status, header, want): + auth = self._make_auth(mock_config) + response = Mock(status_code=status, headers={"Retry-After": header}) + response.json.return_value = {} + auth.client.post.return_value = response + assert auth.poll_device_token("dc", "https://id.example.test").retry_after == want + + def test_wait_polls_no_faster_than_6_s_and_backs_off(self, mock_config, monkeypatch): + from deepctl_core import auth as auth_module + from deepctl_core.auth import DeviceCodeResponse, DevicePoll, TokenResponse + + auth = self._make_auth(mock_config) + slept = [] + monkeypatch.setattr(auth_module.time, "sleep", slept.append) + answers = [ + DevicePoll("pending"), + DevicePoll("unavailable", retry_after=20), + DevicePoll("slow_down"), + DevicePoll("token", token=TokenResponse(access_token="j", dg_token="k")), + ] + monkeypatch.setattr(auth, "poll_device_token", lambda *a: answers.pop(0)) + auth._poll_for_token( + DeviceCodeResponse( + device_code="dc", + verification_uri="https://id.example.test/auth/cli", + expires_in=600, + interval=1, + ), + "https://id.example.test", + ) + assert slept == [6, 20, 25] + + def test_terminal_flow_prints_the_link_and_code_line(self, mock_config, monkeypatch, capsys): + """J2.3: the person's flow shows I-LOGIN-PENDING on stderr.""" + from deepctl_core import auth as auth_module + from deepctl_core import output + from deepctl_core.auth import DeviceCodeResponse, TokenResponse + + auth = self._make_auth(mock_config) + monkeypatch.setattr(auth_module, "check_identity_host", lambda c: "https://id.example.test") + monkeypatch.setattr( + auth, + "_request_device_code", + lambda url: DeviceCodeResponse( + device_code="dc", + user_code="WDJB-MJHT", + verification_uri="https://id.example.test/auth/cli", + expires_in=600, + ), + ) + monkeypatch.setattr(auth, "_poll_for_token", lambda *a: TokenResponse(access_token="j")) + monkeypatch.setattr(auth, "_store_token", lambda t: None) + monkeypatch.setattr(auth_module.webbrowser, "open", lambda url: True) + monkeypatch.setitem(output._output_config, "agentic", False) + monkeypatch.setitem(output._output_config, "quiet", False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + for open_browser in (True, False): + auth.login_with_device_flow(open_browser=open_browser) + err = capsys.readouterr().err + assert ( + "To finish logging in, open https://id.example.test/auth/cli and " + "enter the code WDJB-MJHT." in err + ) + assert "Enter code" not in err and "didn't open" not in err + assert ("Opening browser" in err) is open_browser + assert ("Open the link below" in err) is (not open_browser) + + def test_device_code_request_sends_no_hostname(self, mock_config): + auth = self._make_auth(mock_config) + response = Mock() + response.status_code = 201 + response.json.return_value = { + "device_code": "dc", + "user_code": "ABCD-EFGH", + "verification_uri": "https://id.example.test/auth/cli", + "expires_in": 300, + "interval": 5, + } + auth.client.post.return_value = response + auth._request_device_code("https://id.example.test") + sent = auth.client.post.call_args.kwargs["json"] + assert sent["hostname"] == "deepctl" + import os as _os + + if hasattr(_os, "uname"): + assert _os.uname().nodename not in json.dumps(sent) + + @pytest.mark.parametrize( + "uri", + ["http://id.example.test/auth/cli", "ftp://x/y", "javascript:alert(1)"], + ) + def test_device_code_link_must_be_https(self, mock_config, uri): + from deepctl_core.auth import AuthenticationError + + auth = self._make_auth(mock_config) + response = Mock() + response.status_code = 201 + response.json.return_value = { + "device_code": "dc", + "user_code": "ABCD-EFGH", + "verification_uri": uri, + "expires_in": 300, + } + auth.client.post.return_value = response + with pytest.raises(AuthenticationError): + auth._request_device_code("https://id.example.test") + + def test_device_code_link_may_be_loopback_http(self, mock_config): + from deepctl_core.auth import DeviceCodeResponse + + d = DeviceCodeResponse( + device_code="dc", + verification_uri="http://127.0.0.1:8080/auth/cli", + expires_in=300, + ) + assert d.interval == 5 # ------------------------------------------------------------------ # logout — clears JWT entries diff --git a/packages/deepctl-core/tests/unit/test_config_login.py b/packages/deepctl-core/tests/unit/test_config_login.py new file mode 100644 index 00000000..1528f122 --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_config_login.py @@ -0,0 +1,36 @@ +"""Config details login relies on: the refusal carries its cause, and the +legacy-config migration never writes to stdout.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +from deepctl_core import config as config_module +from deepctl_core.config import Config, ConfigSaveRefused + + +def test_save_refusal_carries_its_cause(tmp_path): + exc = config_module._save_refused("writable", tmp_path / "c.yaml") + assert isinstance(exc, ConfigSaveRefused) + assert exc.why == "writable" + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX homes") +def test_legacy_migration_prints_to_stderr_only(tmp_path, monkeypatch, capsys): + home = tmp_path / "home" + old = home / ".config" / "deepgram" + old.mkdir(parents=True) + (old / "config.yaml").write_text("output:\n format: json\n") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setattr(Path, "home", staticmethod(lambda: home)) + monkeypatch.setattr("deepctl_core.output.is_ai_agent", lambda: False) + monkeypatch.chdir(tmp_path) + Config() + captured = capsys.readouterr() + assert captured.out == "" + assert "Migrating config" in captured.err + assert os.path.exists(home / ".config" / "deepctl" / "config.yaml") diff --git a/packages/deepctl-core/tests/unit/test_login_state.py b/packages/deepctl-core/tests/unit/test_login_state.py new file mode 100644 index 00000000..c8f84dbd --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_login_state.py @@ -0,0 +1,186 @@ +"""The state dir helper and the pending device-login store.""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from pathlib import Path + +import pytest +from deepctl_core import login_state +from deepctl_core.login_state import ( + PendingLogin, + load_pending, + remove_pending, + save_pending, + unwritable_dir, +) + +posix = pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") + + +@pytest.fixture +def state(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setattr(Path, "home", staticmethod(lambda: home)) + return home / ".deepctl" + + +def pending(**over): + values = dict( + device_code="dc", + user_code="ABCD-EFGH", + verification_uri="https://id.example/auth/cli", + interval=6, + expires_at=2_000_000_000.0, + host="https://id.example", + profile="default", + ) + values.update(over) + return PendingLogin(**values) + + +def test_state_dir_is_dot_deepctl_in_home(state): + assert login_state.state_dir() == state + + +@posix +@pytest.mark.parametrize("umask", [0o022, 0o002, 0o077, 0]) +def test_saved_file_is_0600_in_a_0700_dir(state, umask): + old = os.umask(umask) + try: + path = save_pending(pending()) + finally: + os.umask(old) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(state.stat().st_mode) == 0o700 + assert json.loads(path.read_text())["device_code"] == "dc" + assert [p.name for p in state.iterdir()] == ["login-pending.json"] + + +def test_save_replaces_atomically(state): + save_pending(pending(device_code="one")) + save_pending(pending(device_code="two")) + assert load_pending(agent=True).pending.device_code == "two" + assert [p.name for p in state.iterdir()] == ["login-pending.json"] + + +@posix +def test_save_never_follows_a_planted_link(state, tmp_path): + state.mkdir(mode=0o700) + target = tmp_path / "victim" + target.write_text("keep") + (state / "login-pending.json").symlink_to(target) + save_pending(pending()) + assert target.read_text() == "keep" + assert not (state / "login-pending.json").is_symlink() + + +@posix +def test_state_dir_owned_by_someone_else_is_refused(state, monkeypatch): + state.mkdir(mode=0o700) + other = os.geteuid() + 1 + monkeypatch.setattr(login_state.os, "geteuid", lambda: other) + with pytest.raises(login_state.StateDirError): + save_pending(pending()) + + +def test_round_trip_and_remove(state): + save_pending(pending()) + loaded = load_pending(agent=True) + assert loaded.pending == pending() and loaded.ignored_why is None + remove_pending(loaded) + assert not loaded.path.exists() + + +def test_remove_keeps_a_newer_file(state): + save_pending(pending(device_code="old")) + loaded = load_pending(agent=True) + save_pending(pending(device_code="new")) + remove_pending(loaded) + assert load_pending(agent=True).pending.device_code == "new" + + +@pytest.mark.parametrize( + "text", + ["", "not json", "[]", '{"device_code": "dc"}', json.dumps({"interval": True})], +) +def test_malformed_file_is_absent(state, text): + state.mkdir() + (state / "login-pending.json").write_text(text) + (state / "login-pending.json").chmod(0o600) # any umask + loaded = load_pending(agent=True) + assert loaded.pending is None and loaded.ignored_why is None + + +@posix +@pytest.mark.parametrize(("mode", "why"), [(0o666, "writable"), (0o620, "writable")]) +def test_agent_ignores_a_file_others_can_change(state, mode, why): + path = save_pending(pending()) + os.chmod(path, mode) + assert load_pending(agent=True).ignored_why == why + assert load_pending(agent=False).pending == pending() + + +@posix +def test_agent_ignores_a_hard_linked_file(state, tmp_path): + path = save_pending(pending()) + os.link(path, tmp_path / "other") + assert load_pending(agent=True).ignored_why == "links" + + +def test_expiry(state): + p = pending(expires_at=100.0) + assert p.expired(now=100.0) and not p.expired(now=99.0) + assert p.expires_in(now=40.0) == 60 + + +@posix +def test_unwritable_dir_checks_the_nearest_existing_parent(tmp_path): + ro = tmp_path / "ro" + ro.mkdir() + os.chmod(ro, 0o500) + try: + if os.access(ro, os.W_OK): + pytest.skip("root ignores directory modes") + assert unwritable_dir(ro / "a" / "b") == ro / "a" / "b" + assert unwritable_dir(tmp_path / "new" / "dir") is None + f = ro / "config.yaml" + assert unwritable_dir(ro, f) == ro + finally: + os.chmod(ro, 0o700) + f.write_text("x") + os.chmod(ro, 0o500) + try: + assert unwritable_dir(ro, f) is None # rewritten in place + finally: + os.chmod(ro, 0o700) + + +def test_a_file_in_the_way_is_unwritable(tmp_path): + (tmp_path / "file").write_text("x") + assert unwritable_dir(tmp_path / "file" / "sub") == tmp_path / "file" / "sub" + + +@posix +def test_unreadable_file_of_someone_else_is_reported(state, monkeypatch): + path = save_pending(pending()) + os.chmod(path, 0) + try: + if os.access(path, os.R_OK): + pytest.skip("root reads any file") + assert load_pending(agent=True).ignored_why is None # our own file + other = os.geteuid() + 1 + monkeypatch.setattr(login_state.os, "geteuid", lambda: other) + assert load_pending(agent=True).ignored_why == "owner" + assert load_pending(agent=False).ignored_why is None + finally: + os.chmod(path, 0o600) + + +def test_interval_below_the_floor_is_raised_on_read(state): + save_pending(pending(interval=1)) + assert load_pending(agent=True).pending.interval == login_state.MIN_POLL_INTERVAL diff --git a/src/deepctl/_manifest.d/login.json b/src/deepctl/_manifest.d/login.json index 700e53ed..4060433f 100644 --- a/src/deepctl/_manifest.d/login.json +++ b/src/deepctl/_manifest.d/login.json @@ -8,7 +8,7 @@ "requires_auth": false, "help": { "name": "login", - "help": "Log in to Deepgram with browser-based authentication or API key\n\nExamples:\n dg login\n dg login --api-key YOUR_API_KEY\n dg login --api-key YOUR_KEY --project-id YOUR_PROJECT\n dg login --profile staging --api-key YOUR_KEY\n dg --base-url https://api.eu.deepgram.com login --save-base-url\n\n\nFor agents, use --agent-friendly.", + "help": "Log in to Deepgram with browser-based authentication or API key\n\nExamples:\n dg login\n dg login --no-browser\n dg login --complete\n dg login --api-key YOUR_API_KEY\n dg login --api-key YOUR_KEY --project-id YOUR_PROJECT\n dg login --profile staging --api-key YOUR_KEY\n dg --base-url https://api.eu.deepgram.com login --save-base-url\n\n\nFor agents, use --agent-friendly.", "short_help": "Log in to Deepgram", "epilog": null, "hidden": false, @@ -37,6 +37,69 @@ "hidden": false, "help": "Save the --base-url URL as this profile's API URL (asks for confirmation in a terminal)" }, + { + "class": "option", + "name": "wait", + "opts": [ + "--wait" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Wait until the browser login is approved (the default in a terminal), instead of exiting with the link and code; with --complete, keep checking" + }, + { + "class": "option", + "name": "complete", + "opts": [ + "--complete" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Finish a pending browser login: check once whether it was approved, and save the key if it was (add --wait to keep checking)" + }, + { + "class": "option", + "name": "no_browser", + "opts": [ + "--no-browser" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Print the login link and code instead of opening a browser, then exit; finish with --complete, or add --wait to wait here" + }, { "class": "option", "name": "profile", From 16aeca8e2ba19641b2c7903a6865ed5d1be38f9a Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 11:27:42 -0400 Subject: [PATCH 03/18] perf(core): drop the pre-flight key check and reuse one HTTP client Every authed command listed projects before its real request, twice (once in BaseCommand.execute and again when the SDK client was built), on a second TCP connection. That cost two round trips per command and 31 s on a stalled network. - guard() checks only that a key is present. The host check still runs first, before any key is read, and the missing-key (auth_required) and E_KEY_UNTRUSTED paths are unchanged. The removed check's error text ("Network error during verification: ...", "Your API key may have expired or been revoked.") is gone. - New deepctl_core.transport: one httpx.Client per process, created on first use and passed to the SDK as httpx_client. It carries no base URL, no credentials and keeps no cookies; DeepgramClient still takes the base URL from hosts.check_credential_host(). Its timeout matches the SDK's own client (60 s). It follows no redirects, as 0.4.1's SDK client doesn't, and a 3xx keeps 0.4.1's refusal. - Request failures map to the catalogue's transport_mapping codes: DNS, connect, TLS, reset, a proxy refusal, a cut-off body (E-NETWORK-CUT) or a 2xx body that won't parse (E-NETWORK-UNREADABLE) -> network_unavailable; timeout -> timeout; 401 auth_invalid; 403 auth_forbidden; 404 not_found; 429 rate_limited; 400, 422, other 4xx invalid_argument; 5xx server_error. The SDK's own retries still run first. Other failures keep their 0.4.0 wording. - BaseCommand renders a mapped failure as two stderr lines (ERROR:/HINT: in mode A, a cross and the hint otherwise) and, under a machine format, a coded result on stdout with code, hint, next_step, schema_version 1 and details; exit 1. details.key_source follows dg whoami's order. The 0.4.1 coded results keep their keys. - dg speak marks stdout as its payload stream once audio starts, so a failure mid-stream goes to stderr only, never as JSON after the audio. - projects --set-default still warns and asks when the key can't see the project (404/403); other request failures stop it. The 0.4.1 host tests that used the pre-flight as their probe now send the command's own request through the shared client. --- .../src/deepctl_cmd_login/command.py | 5 + .../tests/unit/test_login_device_agent.py | 19 + .../src/deepctl_cmd_projects/command.py | 10 +- .../tests/unit/test_projects_command.py | 35 + .../src/deepctl_cmd_speak/command.py | 6 + .../deepctl-core/src/deepctl_core/auth.py | 25 +- .../src/deepctl_core/base_command.py | 38 +- .../deepctl-core/src/deepctl_core/client.py | 44 +- .../src/deepctl_core/transport.py | 624 +++++++++++++++ packages/deepctl-core/tests/unit/test_auth.py | 25 +- .../tests/unit/test_base_url_paths.py | 17 +- .../tests/unit/test_base_url_provenance.py | 62 +- .../deepctl-core/tests/unit/test_client.py | 2 + .../deepctl-core/tests/unit/test_transport.py | 729 ++++++++++++++++++ 14 files changed, 1570 insertions(+), 71 deletions(-) create mode 100644 packages/deepctl-core/src/deepctl_core/transport.py create mode 100644 packages/deepctl-core/tests/unit/test_transport.py diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index 0f073e7e..32a369e5 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1100,6 +1100,7 @@ def _complete( """``dg login --complete``: poll once (or, with --wait, until done) for the saved device code, and save the key once it is approved.""" import httpx + from deepctl_core import transport stop = _preflight(config) if stop is not None: @@ -1145,6 +1146,10 @@ def _complete( try: poll = auth_manager.poll_device_token(pending.device_code, identity_url) except httpx.RequestError as e: + # The coded network error every other request gets. + mapped = transport.map_exception(e, (pending.device_code,)) + if mapped is not None: + raise mapped from e text = f"Network error while completing the login: {e}" print_error(escape(text)) return LoginResult( diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py b/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py index 9d6f32d6..df0f3341 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_device_agent.py @@ -696,6 +696,25 @@ def test_small_pending_interval_is_raised_to_the_floor(env): assert result.details["retry_after"] == 6 +def test_network_error_on_complete_is_coded(env, monkeypatch): + import httpx + from deepctl_core.transport import TransportError + + run(env) + + def unreachable(self, device_code, identity_url=None): + request = httpx.Request("POST", f"{IDENTITY}/oauth/token") + raise httpx.ConnectError("[Errno 111] Connection refused", request=request) + + monkeypatch.setattr(AuthManager, "poll_device_token", unreachable) + with pytest.raises(TransportError) as err: + run(env, complete=True) + assert err.value.code == "network_unavailable" + assert err.value.error == "Couldn't reach id.dx.deepgram.com: connection refused." + assert "device-secret-1" not in err.value.error + assert pending_path(env).exists() + + def test_complete_with_another_profile_is_a_usage_error(env): import click diff --git a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py index 5ee065d3..b34b149c 100644 --- a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py +++ b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py @@ -11,6 +11,7 @@ get_output_format, get_status_console, ) +from deepctl_core.transport import TransportError from rich.console import Console from .models import ProjectInfo, ProjectsResult @@ -321,7 +322,14 @@ def _set_default_project( try: client.get_project(project_id) status_console.print("[green]✓[/green] Project ID validated") - except Exception as e: + except (Exception, TransportError) as e: + # A project the key can't see (404/403) can still be set, + # after a warning; any other failure stops here. + if isinstance(e, TransportError) and e.code not in ( + "not_found", + "auth_forbidden", + ): + raise status_console.print( f"[yellow]Warning:[/yellow] Could not validate project: {e}" ) diff --git a/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py b/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py index 6e6e539a..67b86260 100644 --- a/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py +++ b/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py @@ -71,3 +71,38 @@ def test_list_default_mode_prints_list(self, mock_console, _fmt, command): ) assert mock_console.print.called + + +class TestSetDefaultValidation: + """--set-default warns and asks when the key can't see the project, as + in 0.4.x; any other request failure stops the command.""" + + @staticmethod + def _run(code): + from deepctl_core.transport import TransportError + + command = ProjectsCommand() + config = Mock(spec=Config) + config.profile = "default" + client = Mock() + client.get_project.side_effect = TransportError( + code, {"path": "/v1/projects/p9", "http_status": 503}, {} + ) + with patch( + "deepctl_cmd_projects.command.DeepgramClient", return_value=client + ), patch.object(command, "confirm", return_value=False) as confirm: + result = command._set_default_project(config, Mock(), "p9") + return result, confirm, config + + @pytest.mark.parametrize("code", ["not_found", "auth_forbidden"]) + def test_unseen_project_warns_and_confirms(self, code): + result, confirm, config = self._run(code) + confirm.assert_called_once() + assert result.status == "cancelled" + config.update_profile.assert_not_called() + + def test_other_failures_stop(self): + from deepctl_core.transport import TransportError + + with pytest.raises(TransportError): + self._run("server_error") diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index c1d696a4..df76e6b7 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -789,6 +789,9 @@ def handle( for chunk in stream: if not chunk: continue + # stdout now carries audio: a later failure is reported + # on stderr only, never as JSON after the audio. + self.stdout_is_protocol = True if eff_encoding == "linear16" and not wrote_header: out.write( _streaming_wav_header(sample_rate=int(eff_sample_rate)) @@ -894,6 +897,9 @@ def handle( # Write to stdout (piping) stdout_buffer = sys.stdout.buffer for chunk in audio_iter: + # stdout now carries audio: a later failure is reported + # on stderr only, never as JSON after the audio. + self.stdout_is_protocol = True stdout_buffer.write(chunk) total_bytes += len(chunk) stdout_buffer.flush() diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index 5261ebcf..03044105 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -579,12 +579,13 @@ def verify_credentials( ) def guard(self) -> None: - """Guard function to ensure a valid API key is available. + """Check that an API key is available, without sending it anywhere. - Only validates that the API key works. Project ID validation - is handled separately by requires_project on BaseCommand. - Refuses first, before reading or sending any key, when the base URL - isn't trusted with credentials. + Refuses first, before reading any key, when the base URL isn't + trusted with credentials. The key itself is checked by the command's + own request: a 401 or 403 there becomes ``auth_invalid`` or + ``auth_forbidden`` (see ``deepctl_core.transport``). Project ID + validation is handled separately by requires_project on BaseCommand. """ self.check_base_url() api_key = self.get_api_key() @@ -613,20 +614,6 @@ def guard(self) -> None: ) raise AuthenticationError("DEEPGRAM_API_KEY is not set") - # Verify the API key is valid (does NOT require project_id) - success, message, error_type = self.verify_api_key(api_key) - if not success: - console.print(f"[red]Error:[/red] {message}") - - if error_type == "auth": - console.print( - "[yellow]Your API key may have expired or been " - "revoked.[/yellow]\n" - "[yellow]Run[/yellow] [bold]deepctl login[/bold] " - "[yellow]to re-authenticate.[/yellow]" - ) - raise AuthenticationError(message) - def login_with_api_key( self, api_key: str, project_id: str | None = None, _force_write: bool = False ) -> None: diff --git a/packages/deepctl-core/src/deepctl_core/base_command.py b/packages/deepctl-core/src/deepctl_core/base_command.py index 14830062..182a198d 100644 --- a/packages/deepctl-core/src/deepctl_core/base_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_command.py @@ -10,9 +10,16 @@ from .config import Config, flush_config_notices from .hosts import CredentialHostError from .models import CodedErrorResult, ErrorResult -from .output import _agentic, print_error, print_error_hint, stderr_console +from .output import ( + _agentic, + print_error, + print_error_hint, + print_hint, + stderr_console, +) from .output import console as stdout_console from .timing import TimingContext +from .transport import TransportError # The PAYLOAD console -- stdout, deliberately. Tables, JSON and raw text # written by _output_* are the machine-readable result and belong there. @@ -210,6 +217,10 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: stderr_console.print("\n[yellow]Command cancelled by user[/yellow]") raise click.Abort() + except TransportError as e: + self._report_transport_error(e, ctx, config, auth_manager) + raise SystemExit(1) from None + except CredentialHostError as e: # Already reported on stderr; nothing was sent. try: @@ -234,6 +245,31 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: if exit_code: raise SystemExit(exit_code) + def _report_transport_error( + self, + error: TransportError, + ctx: click.Context, + config: Config, + auth_manager: AuthManager, + ) -> None: + """Two stderr lines, and the coded result on stdout.""" + from rich.markup import escape + + from . import transport + + command_path = " ".join(ctx.command_path.split()[1:]) + result = transport.render( + error, command_path=command_path, auth_manager=auth_manager + ) + print_error(escape(result.error)) + print_hint(escape(result.hint)) + if self.stdout_is_protocol: + return + try: + self.output_result(result, config) + except (OSError, ValueError): + pass + def pre_auth_check(self, config: Config, **kwargs: Any) -> Any | None: """Return a result to stop with before authentication, or ``None``. diff --git a/packages/deepctl-core/src/deepctl_core/client.py b/packages/deepctl-core/src/deepctl_core/client.py index aeec1f37..19399c33 100644 --- a/packages/deepctl-core/src/deepctl_core/client.py +++ b/packages/deepctl-core/src/deepctl_core/client.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlencode +from . import transport from .hosts import DEFAULT_BASE_URL from .output import stderr_console @@ -91,6 +92,8 @@ def __init__(self, config: Config, auth_manager: AuthManager): self._project_id: str | None = None # The REST base the SDK client was given (for redirect errors). self._rest_base: str = DEFAULT_BASE_URL + # Kept out of error text. + self._secrets: tuple[str, ...] = () @property def client(self) -> DGClient: @@ -124,7 +127,11 @@ def _create_client(self) -> DGClient: base_url = self.auth_manager.check_base_url() try: - kwargs: dict[str, Any] = {"api_key": api_key} + # One connection pool per process; requests reuse its connection. + kwargs: dict[str, Any] = { + "api_key": api_key, + "httpx_client": transport.shared_client(), + } if base_url and base_url != DEFAULT_BASE_URL: # Derive both the REST and WebSocket endpoints from the @@ -149,6 +156,7 @@ def _create_client(self) -> DGClient: client = DGClient(**kwargs) self._project_id = project_id + self._secrets = (api_key,) return client except Exception as e: @@ -174,13 +182,15 @@ def _failed(self, what: str, e: Exception) -> Exception: ) return ApiError(body=f"{what}: {e}") - def _refuse_redirects_in(self, chunks: Iterator[bytes]) -> Iterator[bytes]: - """Iterate ``chunks``, turning a 3xx answer into the refusal.""" + def _mapped(self, chunks: Iterator[bytes]) -> Iterator[bytes]: + """Yield ``chunks``; a request failure while reading them is mapped, + and a 3xx answer becomes the redirect refusal.""" from .websocket import RedirectRefused try: yield from chunks except Exception as e: + transport.raise_for(e, self._secrets) refused = self._failed("", e) if isinstance(refused, RedirectRefused): raise refused from e @@ -216,6 +226,7 @@ def transcribe_file( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Transcription failed", e) def transcribe_url( @@ -237,6 +248,7 @@ def transcribe_url( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Transcription failed", e) # ── Text-to-Speech ───────────────────────────────────────────── @@ -263,9 +275,10 @@ def speak_text( "Iterator[bytes]", self.client.speak.v1.audio.generate(**kwargs) ) # The request runs when the caller starts iterating. - return self._refuse_redirects_in(audio) + return self._mapped(audio) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Text-to-speech failed", e) def speak_text_stream( @@ -326,6 +339,7 @@ def speak_text_stream( if isinstance(message, bytes): yield message except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Text-to-speech streaming failed", e) def ws_base_url(self) -> str: @@ -368,6 +382,7 @@ def analyze_text( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Text analysis failed", e) # ── Models ───────────────────────────────────────────────────── @@ -383,6 +398,7 @@ def list_models(self, include_outdated: bool = False) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list models", e) def get_model(self, model_id: str) -> dict[str, Any]: @@ -392,6 +408,7 @@ def get_model(self, model_id: str) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get model", e) # ── Projects ─────────────────────────────────────────────────── @@ -403,6 +420,7 @@ def get_projects(self) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get projects", e) def get_project(self, project_id: str | None = None) -> dict[str, Any]: @@ -413,6 +431,7 @@ def get_project(self, project_id: str | None = None) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get project", e) def create_project(self, name: str, company: str | None = None) -> dict[str, Any]: @@ -426,6 +445,7 @@ def create_project(self, name: str, company: str | None = None) -> dict[str, Any return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to create project", e) # ── Usage ────────────────────────────────────────────────────── @@ -451,6 +471,7 @@ def get_usage( return result except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get usage", e) # ── API Keys ─────────────────────────────────────────────────── @@ -471,6 +492,7 @@ def list_keys( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list keys", e) def create_key( @@ -502,6 +524,7 @@ def create_key( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to create key", e) def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: @@ -512,6 +535,7 @@ def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get key", e) def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: @@ -522,6 +546,7 @@ def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, An return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to delete key", e) # ── Requests ─────────────────────────────────────────────────── @@ -560,6 +585,7 @@ def list_requests( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list requests", e) def get_request( @@ -572,6 +598,7 @@ def get_request( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get request", e) # ── Billing ──────────────────────────────────────────────────── @@ -600,6 +627,7 @@ def get_billing_breakdown( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get billing breakdown", e) def get_balances(self, project_id: str | None = None) -> dict[str, Any]: @@ -610,6 +638,7 @@ def get_balances(self, project_id: str | None = None) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to get balances", e) # ── Members ──────────────────────────────────────────────────── @@ -622,6 +651,7 @@ def list_members(self, project_id: str | None = None) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list members", e) def remove_member( @@ -634,6 +664,7 @@ def remove_member( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to remove member", e) def list_member_scopes( @@ -648,6 +679,7 @@ def list_member_scopes( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list member scopes", e) def update_member_scopes( @@ -662,6 +694,7 @@ def update_member_scopes( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to update member scopes", e) # ── Invites ──────────────────────────────────────────────────── @@ -674,6 +707,7 @@ def list_invites(self, project_id: str | None = None) -> dict[str, Any]: return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to list invites", e) def create_invite( @@ -688,6 +722,7 @@ def create_invite( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to create invite", e) def delete_invite( @@ -700,6 +735,7 @@ def delete_invite( return cast("dict[str, Any]", response.model_dump()) except Exception as e: + transport.raise_for(e, self._secrets) raise self._failed("Failed to delete invite", e) # ── Utilities ────────────────────────────────────────────────── diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py new file mode 100644 index 00000000..70717dc5 --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -0,0 +1,624 @@ +"""One HTTP client per process, and the map from transport failures to codes. + +Every REST call the SDK facade makes goes through :func:`shared_client`: one +``httpx.Client`` (one connection pool) per process, created on first use so +``--version`` and ``--help`` never import httpx. The client carries no +credentials and no base URL. The key and the host still come from +``DeepgramClient``, which takes the base URL from +:func:`deepctl_core.hosts.check_credential_host` before any request carries +the key; this module never decides where a credential may go. + +:func:`raise_for` turns a failed request into a :class:`TransportError` with a +catalogue code (``transport_mapping`` in ``error_codes.json``, in +order): DNS, connect, TLS, reset or a truncated 2xx body -> +``network_unavailable``; timeout -> ``timeout``; 401 -> ``auth_invalid``; +403 -> ``auth_forbidden``; 404 -> ``not_found``; 429 -> ``rate_limited``; +400, 422 and any other 4xx -> ``invalid_argument``; 5xx -> ``server_error``. +``BaseCommand.execute`` renders it: a coded result on stdout under a machine +format, and two stderr lines. +""" + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import sys +import threading +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import httpx + + from .auth import AuthManager + +# The SDK's own default for a client it builds itself. +DEFAULT_TIMEOUT_S = 60.0 + +_lock = threading.Lock() +_client: httpx.Client | None = None +# Where the most recent request went, for the error text: method, host and +# path only (never the query string, which can carry values from argv). +_last_request: dict[str, str] = {} + + +def _record_request(request: httpx.Request) -> None: + _last_request.clear() + _last_request.update( + method=request.method, host=request.url.host, path=request.url.path + ) + + +def _install(client: httpx.Client) -> httpx.Client: + _no_cookies(client) + hooks = client.event_hooks + hooks.setdefault("request", []) + if _record_request not in hooks["request"]: + hooks["request"] = [*hooks["request"], _record_request] + client.event_hooks = hooks + return client + + +def shared_client() -> httpx.Client: + """The process's one ``httpx.Client``, created on first use. + + It follows no redirects: a 3xx could take the key, or a request body, + to another host (see ``DeepgramClient._failed``). + """ + global _client + with _lock: + if _client is None: + import httpx + + _client = _install( + httpx.Client(timeout=DEFAULT_TIMEOUT_S, follow_redirects=False) + ) + return _client + + +def _no_cookies(client: httpx.Client) -> None: + """Keep no cookies: one process can serve several profiles and hosts.""" + from http.cookiejar import DefaultCookiePolicy + + client.cookies.jar.set_policy(DefaultCookiePolicy(allowed_domains=[])) + + +def _set_client(client: httpx.Client | None) -> None: + """Replace the shared client (tests pass one with an ``httpx.MockTransport``).""" + global _client + with _lock: + _client = _install(client) if client is not None else None + _last_request.clear() + + +# ── Catalogue text (error_codes.json "strings") ────────────────────── + +STRINGS: dict[str, str] = { + "E-AUTH-INVALID": "Deepgram rejected the API key (HTTP 401).", + "H-AUTH-INVALID": ( + "The API key may have expired or been revoked. Run {prog} login to log " + "in again." + ), + "A-AUTH-INVALID": ( + "The API key was rejected. Run {prog} login and give the user the " + "verification link and code it prints." + ), + "A-AUTH-INVALID-ENV": ( + "The DEEPGRAM_API_KEY in this environment was rejected. next_step starts " + "a login whose saved key will take precedence over it; give the user the " + "link and code it prints." + ), + "A-AUTH-INVALID-ENV-NULL": ( + "The DEEPGRAM_API_KEY in this environment was rejected. Ask the user to " + "fix the key, or to run {prog} login --yes in their terminal." + ), + "C-AUTH-INVALID": ( + "The API key was rejected. Set a valid DEEPGRAM_API_KEY in this job's " + "environment." + ), + "E-AUTH-FORBIDDEN": ("This API key isn't allowed to make this request (HTTP 403)."), + "H-AUTH-FORBIDDEN": ( + "Use an API key with the needed permissions, or run {prog} login with " + "an account that has access." + ), + "A-AUTH-FORBIDDEN": ( + "The API key lacks a permission this request needs. Tell the user; " + "they can run {prog} login with an account that has access." + ), + "E-NETWORK": "Couldn't reach {host}: {reason}.", + "H-NETWORK": "Check your network connection and proxy settings, then try again.", + "A-NETWORK": ( + "{prog} can't reach Deepgram from this environment. If you run in a " + "sandbox, ask the user to allow network access for {prog}." + ), + "E-NETWORK-CUT": "The connection to {host} dropped before the response finished.", + "H-NETWORK-CUT": ( + "Try again. If it keeps happening, a proxy or firewall may be cutting off " + "responses." + ), + "A-NETWORK-CUT": ( + "Retry once. If it fails again, tell the user the connection to Deepgram " + "keeps dropping." + ), + "E-NETWORK-UNREADABLE": "Got a response from {host} that {prog} couldn't read.", + "H-NETWORK-UNREADABLE": ( + "A proxy, captive portal or firewall may be answering in Deepgram's " + "place. Check that this network lets {prog} reach {host}, then try again." + ), + "A-NETWORK-UNREADABLE": ( + "Retry once. If it fails again, tell the user that something on this " + "network (a proxy, captive portal or firewall) is answering in " + "Deepgram's place." + ), + "E-TIMEOUT": "{host} didn't respond within {timeout_s} s.", + "H-TIMEOUT": "Try again. Large files take longer to upload on slow connections.", + "A-TIMEOUT": ( + "Try again once. If it times out again, tell the user the network is " + "slow or blocked." + ), + "E-RATE-LIMITED": "Deepgram is rate limiting requests (HTTP 429).", + "H-RATE-LIMITED": "Wait a moment and try again.", + "A-RATE-LIMITED": ( + "Wait the number of seconds in details.retry_after (or 10 s if absent) " + "before retrying." + ), + "E-NOT-FOUND": "Deepgram returned 404 Not Found for {path}.", + "H-NOT-FOUND": "Check the ID, and that it belongs to the selected project.", + "A-NOT-FOUND": ( + "Check the ID. List the available items first instead of guessing IDs." + ), + "E-INVALID-REQUEST": ( + "Deepgram rejected the request (HTTP {http_status}): {api_message}" + ), + "H-INVALID-REQUEST": ( + "Check the options you passed. Run {prog} {command_path} --help to see " + "valid values." + ), + "A-INVALID-REQUEST": ( + "Fix the option named in the message and retry. Run {prog} " + "{command_path} --help for valid values." + ), + "E-SERVER": "Deepgram returned an error (HTTP {http_status}).", + "H-SERVER": ( + "Try again in a moment. If it keeps failing, contact Deepgram support " + "with the request ID in details." + ), + "A-SERVER": ( + "Retry once after a few seconds. If it fails again, tell the user and " + "include details.request_id." + ), +} + +# code -> (error message ID, human hint, agent hint, automation hint or None) +Messages = tuple[str, str, str, "str | None"] +CODES: dict[str, Messages] = { + "auth_invalid": ( + "E-AUTH-INVALID", + "H-AUTH-INVALID", + "A-AUTH-INVALID", + "C-AUTH-INVALID", + ), + "auth_forbidden": ( + "E-AUTH-FORBIDDEN", + "H-AUTH-FORBIDDEN", + "A-AUTH-FORBIDDEN", + None, + ), + "network_unavailable": ("E-NETWORK", "H-NETWORK", "A-NETWORK", None), + "timeout": ("E-TIMEOUT", "H-TIMEOUT", "A-TIMEOUT", None), + "rate_limited": ("E-RATE-LIMITED", "H-RATE-LIMITED", "A-RATE-LIMITED", None), + "not_found": ("E-NOT-FOUND", "H-NOT-FOUND", "A-NOT-FOUND", None), + "invalid_argument": ( + "E-INVALID-REQUEST", + "H-INVALID-REQUEST", + "A-INVALID-REQUEST", + None, + ), + "server_error": ("E-SERVER", "H-SERVER", "A-SERVER", None), +} + +# network_unavailable's other messages: the response was cut off, or it +# arrived complete but isn't Deepgram's (a proxy or captive portal page). +NETWORK_CUT: Messages = ("E-NETWORK-CUT", "H-NETWORK-CUT", "A-NETWORK-CUT", None) +NETWORK_UNREADABLE: Messages = ( + "E-NETWORK-UNREADABLE", + "H-NETWORK-UNREADABLE", + "A-NETWORK-UNREADABLE", + None, +) + +# Network failure reasons for E-NETWORK's {reason}. +REASON_DNS = "name resolution failed" +REASON_REFUSED = "connection refused" +REASON_TLS = "TLS handshake failed" +REASON_RESET = "connection reset" +REASON_CLOSED = "connection closed" +REASON_TRUNCATED = "response cut off" +REASON_UNREADABLE = "unreadable response" +REASON_PROXY = "the proxy refused the connection" +REASON_OTHER = "connection failed" + +KEY_SOURCE_ENV = "DEEPGRAM_API_KEY (env)" +KEY_SOURCE_FLAG = "--api-key" +KEY_SOURCE_KEYRING = "system keyring" +KEY_SOURCE_FILE = "config file" + + +class TransportError(BaseException): + """A request failed at the transport or with an HTTP error status. + + A ``BaseException`` on purpose: command handlers wrap their work in + ``except Exception`` and turn anything into a legacy error result. This + one has to reach ``BaseCommand.execute``, which prints the coded error. + Anything else that catches it should re-raise it. + """ + + def __init__( + self, + code: str, + values: dict[str, Any], + details: dict[str, Any], + messages: Messages | None = None, + ) -> None: + self.code = code + self.values = values + self.details = details + self.messages = messages or CODES[code] + super().__init__(self.error) + + @property + def error(self) -> str: + return STRINGS[self.messages[0]].format(**self.values) + + +# ── Mapping ────────────────────────────────────────────────────────── + +_SECRET_URL = re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]*@") +_QUERY = re.compile(r"\?\S*") + + +def _clean(text: object, secrets: tuple[str, ...] = ()) -> str: + """Untrusted text for ``error``.""" + from . import messages + + value = str(text) + for secret in secrets: + if secret: + value = value.replace(secret, "****") + value = _SECRET_URL.sub(r"\1", value) + value = _QUERY.sub("", value) + return messages.display(" ".join(value.split())) + + +def _chain(exc: BaseException) -> list[BaseException]: + seen: list[BaseException] = [] + current: BaseException | None = exc + while current is not None and current not in seen: + seen.append(current) + current = current.__cause__ or current.__context__ + return seen + + +def _network_reason(exc: BaseException) -> str: + import socket + import ssl + + import httpx + + chain = _chain(exc) + for item in chain: + if isinstance(item, socket.gaierror): + return REASON_DNS + if isinstance(item, ssl.SSLError): + return REASON_TLS + if isinstance(item, ConnectionRefusedError): + return REASON_REFUSED + if isinstance(item, ConnectionResetError): + return REASON_RESET + text = " ".join(str(item) for item in chain).lower() + if "name or service not known" in text or "nodename nor servname" in text: + return REASON_DNS + if "temporary failure in name resolution" in text or "getaddrinfo" in text: + return REASON_DNS + if "ssl" in text or "certificate" in text or "tls" in text: + return REASON_TLS + if "connection refused" in text: + return REASON_REFUSED + if isinstance(exc, httpx.RemoteProtocolError): + if "incomplete" in text or "message body" in text: + return REASON_TRUNCATED + return REASON_CLOSED + if isinstance(exc, (httpx.ReadError, httpx.WriteError)) or "reset" in text: + return REASON_RESET + return REASON_OTHER + + +def _host(exc: BaseException | None = None) -> str: + host = None + try: + host = exc.request.url.host # type: ignore[union-attr] + except (AttributeError, RuntimeError): + host = _last_request.get("host") + return _clean(host or "the Deepgram API") + + +def _header(headers: object, name: str) -> str | None: + if not isinstance(headers, dict): + return None + for key, value in headers.items(): + if str(key).lower() == name: + return str(value) + return None + + +def _api_message(body: object, status: int, secrets: tuple[str, ...]) -> str: + text: object = None + if isinstance(body, dict): + for key in ("err_msg", "message", "error", "detail"): + if body.get(key): + text = body[key] + break + elif isinstance(body, str) and body.strip(): + text = body + if text is None: + from http import HTTPStatus + + try: + text = HTTPStatus(status).phrase + except ValueError: + text = "no details" + return _clean(text, secrets) + + +def map_exception( + exc: BaseException, secrets: tuple[str, ...] = () +) -> TransportError | None: + """The coded error for a failed request, or ``None`` when ``exc`` isn't one.""" + if isinstance(exc, TransportError): + return exc + if "httpx" in sys.modules: + import httpx + + if isinstance(exc, httpx.TimeoutException): + host = _host(exc) + return TransportError( + "timeout", + {"host": host, "timeout_s": _seconds(DEFAULT_TIMEOUT_S)}, + {"host": host, "timeout_s": _seconds(DEFAULT_TIMEOUT_S)}, + ) + if isinstance(exc, httpx.ProxyError): + host = _host(exc) + return TransportError( + "network_unavailable", + {"host": host, "reason": REASON_PROXY}, + {"host": host, "reason": REASON_PROXY}, + ) + if isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError)): + host = _host(exc) + reason = _network_reason(exc) + cut = reason == REASON_TRUNCATED + return TransportError( + "network_unavailable", + {"host": host, "reason": reason}, + {"host": host, "reason": reason}, + NETWORK_CUT if cut else None, + ) + + status = getattr(exc, "status_code", None) + if not isinstance(status, int) or "deepgram" not in sys.modules: + return None + from deepgram.core.api_error import ApiError + + if not isinstance(exc, ApiError): + return None + headers = exc.headers + if 200 <= status < 300: + # The SDK got a complete success response it couldn't parse: most + # often not Deepgram's at all (a proxy's or captive portal's page). + # A body cut short raises RemoteProtocolError above instead. + host = _host() + return TransportError( + "network_unavailable", + {"host": host, "reason": REASON_UNREADABLE, "prog": prog() or "dg"}, + {"host": host, "reason": REASON_UNREADABLE}, + NETWORK_UNREADABLE, + ) + if status == 401: + return TransportError("auth_invalid", {}, {"http_status": 401}) + if status == 403: + return TransportError("auth_forbidden", {}, {"http_status": 403}) + if status == 404: + path = _clean(_last_request.get("path") or "this request") + return TransportError( + "not_found", {"path": path}, {"http_status": 404, "path": path} + ) + if status == 429: + details: dict[str, Any] = {"http_status": 429} + retry_after = _header(headers, "retry-after") + if retry_after and retry_after.strip().isdigit(): + details["retry_after"] = int(retry_after.strip()) + return TransportError("rate_limited", {}, details) + if 400 <= status < 500: + return TransportError( + "invalid_argument", + { + "http_status": status, + "api_message": _api_message(exc.body, status, secrets), + }, + {"http_status": status}, + ) + if 500 <= status < 600: + details = {"http_status": status} + request_id = _header(headers, "dg-request-id") + if request_id: + details["request_id"] = _clean(request_id) + return TransportError("server_error", {"http_status": status}, details) + return None + + +def _seconds(value: float) -> int | float: + return int(value) if float(value).is_integer() else value + + +def raise_for(exc: BaseException, secrets: tuple[str, ...] = ()) -> None: + """Raise the coded :class:`TransportError` for ``exc``, if it maps to one.""" + mapped = map_exception(exc, secrets) + if mapped is not None and mapped is not exc: + raise mapped from exc + if mapped is not None: + raise mapped + + +# ── Rendering ──────────────────────────────────────────────────────── + +_PROG_NAMES = ("dg", "deepgram", "deepctl") + + +def prog() -> str | None: + """``{prog}``: the running executable, as a command. + + The bare name when ``which(name)`` is this same file; else the console + script's absolute path, quoted. ``None`` when neither applies (``python + -m``, an unknown launcher), which makes ``next_step`` null. + A stable command name replaces the fallback in a later release. + """ + argv0 = sys.argv[0] if sys.argv else "" + name = os.path.basename(argv0) + if os.name == "nt" and name.lower().endswith(".exe"): + name = name[:-4] + if name not in _PROG_NAMES: + return None + found = shutil.which(name) + running = shutil.which(argv0) or argv0 + try: + if found and os.path.samefile(found, running): + return name + except OSError: + pass + if not os.path.isfile(running): + return None + path = os.path.abspath(running) + if os.name == "nt": + return None if shlex.quote(path) != path else path + return shlex.quote(path) + + +def _key_source(auth_manager: AuthManager | None) -> str | None: + if auth_manager is None: + return None + if auth_manager.explicit_api_key: + return KEY_SOURCE_FLAG + try: + key = auth_manager.get_api_key() + except Exception: + return None + if not key: + return None + # The order `dg whoami` uses: keyring, then the config file (a profile key + # equal to the env value came from the env), then the env. + import keyring + + from .auth import KEYRING_SERVICE + + env_key = os.environ.get("DEEPGRAM_API_KEY") + profile = auth_manager.config.profile or "default" + try: + stored = keyring.get_password(KEYRING_SERVICE, f"api-key.{profile}") + except Exception: + stored = None + if stored and stored == key: + return KEY_SOURCE_KEYRING + # Config merges the env key into the profile, so a file key equal to the + # env value came from the env. + return KEY_SOURCE_ENV if env_key == key else KEY_SOURCE_FILE + + +def render( + err: TransportError, + *, + command_path: str, + auth_manager: AuthManager | None = None, +) -> Any: + """The coded result for ``err``.""" + from . import output + + agentic = bool(output._output_config["agentic"]) + hard = output.is_ai_agent() + soft_only = agentic and not hard + _, human, agent, automation = err.messages + program = prog() + details = dict(err.details) + + next_step: str | None = None + hint_id = human if not agentic else (automation or agent) if soft_only else agent + if err.code == "auth_invalid": + source = _key_source(auth_manager) + if source: + details["key_source"] = source + if not soft_only and program: + next_step = f"{program} login" + if hard and source == KEY_SOURCE_ENV: + if program: + next_step = f"{program} login --yes" + hint_id = "A-AUTH-INVALID-ENV" + else: + hint_id = "A-AUTH-INVALID-ENV-NULL" + elif err.code == "invalid_argument" and program and command_path: + candidate = f"{program} {command_path} --help" + if shlex.join(shlex.split(candidate)) == candidate: + next_step = candidate + + values = { + **err.values, + "prog": program or _PROG_NAMES[0], + "command_path": command_path, + } + hint = STRINGS[hint_id].format(**values) + return result_class()( + error=err.error, + code=err.code, + hint=hint, + next_step=next_step, + details=details or None, + ) + + +def _result_class() -> Any: + from pydantic import model_serializer + + from .models import CodedErrorResult + + class _TransportErrorResult(CodedErrorResult): + """A 0.5.0 coded error: ``error``, ``status``, ``code``, ``hint``, + ``next_step``, ``schema_version`` and, when set, ``details``. + + Only transport errors use it: the 0.4.1 coded errors keep their keys. + """ + + next_step: str | None = None + schema_version: int = 1 + details: dict[str, Any] | None = None + + @model_serializer(mode="wrap") + def _drop_empty_details(self, handler: Any) -> dict[str, Any]: + data: dict[str, Any] = handler(self) + if not data.get("details"): + data.pop("details", None) + return data + + _TransportErrorResult.__name__ = "TransportErrorResult" + _TransportErrorResult.__qualname__ = "TransportErrorResult" + return _TransportErrorResult + + +_RESULT_CLASS: Any = None + + +def result_class() -> Any: + """``TransportErrorResult``, built on first use so pydantic loads only + when a transport error is rendered.""" + global _RESULT_CLASS + if _RESULT_CLASS is None: + _RESULT_CLASS = _result_class() + return _RESULT_CLASS diff --git a/packages/deepctl-core/tests/unit/test_auth.py b/packages/deepctl-core/tests/unit/test_auth.py index 6554b56b..946f1e11 100644 --- a/packages/deepctl-core/tests/unit/test_auth.py +++ b/packages/deepctl-core/tests/unit/test_auth.py @@ -266,22 +266,19 @@ def test_guard_with_no_api_key(self, auth_manager): ): auth_manager.guard() - def test_guard_with_invalid_credentials(self, auth_manager): - """Test guard method with invalid credentials.""" - # Set up API key + def test_guard_sends_nothing(self, auth_manager): + """guard() only checks that a key is there: no pre-flight request. + + An invalid key is reported by the command's own request (401 -> + auth_invalid, deepctl_core.transport).""" with patch.object( auth_manager, "get_api_key", return_value="sk-test-key" - ): - # Mock failed key verification - with patch.object( - auth_manager, - "verify_api_key", - return_value=(False, "Invalid API key", "auth"), - ): - with pytest.raises( - AuthenticationError, match="Invalid API key" - ): - auth_manager.guard() + ), patch.object(auth_manager, "verify_api_key") as verify, patch.object( + auth_manager, "client" + ) as http: + auth_manager.guard() + verify.assert_not_called() + assert http.method_calls == [] def test_guard_does_not_require_project_id(self, auth_manager): """Test that guard() only checks API key, not project_id.""" diff --git a/packages/deepctl-core/tests/unit/test_base_url_paths.py b/packages/deepctl-core/tests/unit/test_base_url_paths.py index 5375e4a2..ef999a4b 100644 --- a/packages/deepctl-core/tests/unit/test_base_url_paths.py +++ b/packages/deepctl-core/tests/unit/test_base_url_paths.py @@ -8,7 +8,7 @@ import httpx import pytest import yaml -from deepctl_core import hosts, output +from deepctl_core import hosts, output, transport from deepctl_core.auth import AuthManager from deepctl_core.client import DeepgramClient from deepctl_core.config import Config @@ -24,6 +24,7 @@ def _isolated(tmp_path, monkeypatch): lambda: tmp_path / "user" / "config.yaml", ) monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(transport, "_client", None) monkeypatch.setattr(output, "is_ai_agent", lambda: False) monkeypatch.setattr(output, "is_agentic", lambda: False) monkeypatch.setattr(output.stderr_console, "_width", 400) @@ -38,12 +39,14 @@ def user(tmp_path, data) -> str: def recording(config): seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"projects": []}) + auth = AuthManager(config) - auth.client = httpx.Client( - transport=httpx.MockTransport( - lambda r: seen.append(r) or httpx.Response(200, json={"projects": []}) - ) - ) + auth.client = httpx.Client(transport=httpx.MockTransport(handler)) + transport._set_client(httpx.Client(transport=httpx.MockTransport(handler))) return auth, seen @@ -61,6 +64,8 @@ def test_env_source(tmp_path, monkeypatch): auth, seen = recording(config) assert auth.check_base_url() == PROXY auth.guard() + assert seen == [] # guard() checks the host and the key, and sends nothing + DeepgramClient(config, auth).get_projects() assert str(seen[0].url) == PROXY + "/v1/projects" diff --git a/packages/deepctl-core/tests/unit/test_base_url_provenance.py b/packages/deepctl-core/tests/unit/test_base_url_provenance.py index cc18cab8..ac720b2a 100644 --- a/packages/deepctl-core/tests/unit/test_base_url_provenance.py +++ b/packages/deepctl-core/tests/unit/test_base_url_provenance.py @@ -13,7 +13,7 @@ import httpx import pytest import yaml -from deepctl_core import hosts, output +from deepctl_core import hosts, output, transport from deepctl_core.auth import AuthManager from deepctl_core.base_command import BaseCommand from deepctl_core.client import DeepgramClient @@ -48,6 +48,7 @@ def _isolated(tmp_path, monkeypatch): "deepctl_core.config._trust_root_location", lambda: (tmp_path, tmp_path / "user" / "config.yaml") ) monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(transport, "_client", None) monkeypatch.setattr(output, "is_agentic", lambda: False) monkeypatch.setattr(output, "is_ai_agent", lambda: False) output._output_config.update(agentic=False, quiet=False, format="default") @@ -79,7 +80,8 @@ def write_user(tmp_path, data) -> str: def recording_auth(config: Config) -> tuple[AuthManager, list[httpx.Request]]: - """AuthManager whose HTTP client records requests instead of sending.""" + """AuthManager whose HTTP clients (its own, and the shared one the SDK + uses) record requests instead of sending.""" seen: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -88,9 +90,17 @@ def handler(request: httpx.Request) -> httpx.Response: auth = AuthManager(config) auth.client = httpx.Client(transport=httpx.MockTransport(handler)) + transport._set_client(httpx.Client(transport=httpx.MockTransport(handler))) return auth, seen +def send(auth: AuthManager) -> None: + """What an authed command does: guard(), then its own request through + the SDK. There is no separate key check.""" + auth.guard() + DeepgramClient(auth.config, auth).get_projects() + + KEYED = {"profiles": {"default": {"api_key": "sk-USERKEY"}}} @@ -250,7 +260,7 @@ def test_agent_rendering_gets_agent_hint(signal, monkeypatch, tmp_path, capsys): monkeypatch.setenv("DEEPGRAM_BASE_URL", "https://proxy.example") config = Config(config_path=write_user(tmp_path, data)) auth, seen = recording_auth(config) - auth.guard() # sent: CI is not an AI-agent marker + send(auth) # sent: CI is not an AI-agent marker assert seen captured = capsys.readouterr() for text in (captured.err, captured.out): @@ -268,11 +278,11 @@ def test_untrusted_text_is_sanitized(): class TestCredentialPaths: - def test_preflight_check_refuses_flag_host(self, tmp_path, capsys): + def test_command_path_refuses_flag_host(self, tmp_path, capsys): config = Config(config_path=write_user(tmp_path, KEYED), base_url=FOREIGN) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError): - auth.guard() + send(auth) assert seen == [] err = capsys.readouterr().err assert err.count("Not sending your API key to evil.example") == 1 @@ -293,11 +303,11 @@ def test_project_check_uses_the_checked_base_url(self, tmp_path, monkeypatch): auth.verify_credentials("sk-USERKEY", "P1") assert [str(r.url) for r in seen] == ["http://127.0.0.1:8080/v1/projects/P1"] - def test_preflight_sends_to_trusted_flag_host(self, tmp_path): + def test_command_path_sends_to_trusted_flag_host(self, tmp_path): data = {**KEYED, "api": {"allowed_hosts": ["evil.example"]}} config = Config(config_path=write_user(tmp_path, data), base_url=FOREIGN) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert [r.url.host for r in seen] == ["evil.example"] assert seen[0].headers["Authorization"] == "Token sk-USERKEY" @@ -360,7 +370,7 @@ def test_project_file_allowlist_does_not_trust_flag_host(self, tmp_path): config = Config(config_path=write_user(tmp_path, KEYED), base_url=FOREIGN) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError) as err: - auth.guard() + send(auth) assert err.value.code == "foreign_host" assert seen == [] @@ -370,7 +380,7 @@ def test_saved_profile_base_url_refused(self, tmp_path): config = Config(config_path=write_user(tmp_path, data)) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError) as err: - auth.guard() + send(auth) assert err.value.message_id == "E-FH-PROFILE" assert seen == [] @@ -381,7 +391,7 @@ def test_saved_profile_base_url_allowlisted(self, tmp_path): } config = Config(config_path=write_user(tmp_path, data)) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert len(seen) == 1 def test_agent_env_host_refused(self, tmp_path, monkeypatch, agentic): @@ -390,7 +400,7 @@ def test_agent_env_host_refused(self, tmp_path, monkeypatch, agentic): config = Config(config_path=write_user(tmp_path, KEYED)) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError) as err: - auth.guard() + send(auth) assert err.value.code == "foreign_host" assert seen == [] @@ -399,14 +409,14 @@ def test_agent_env_host_allowlisted(self, tmp_path, monkeypatch, agentic): data = {**KEYED, "api": {"allowed_hosts": ["evil.example"]}} config = Config(config_path=write_user(tmp_path, data)) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen[0].headers["Authorization"] == "Token sk-USERKEY" def test_non_agent_loopback_env_allowed(self, tmp_path, monkeypatch): monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080") config = Config(config_path=write_user(tmp_path, KEYED)) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen[0].url.host == "127.0.0.1" assert seen[0].headers["Authorization"] == "Token sk-USERKEY" @@ -501,8 +511,8 @@ def test_ci_proxy_env_host_sent_with_one_warning( monkeypatch.setenv("DEEPGRAM_BASE_URL", "https://proxy.example") config = Config(config_path=write_user(tmp_path, KEYED)) auth, seen = recording_auth(config) - auth.guard() - auth.guard() + send(auth) + send(auth) assert [r.url.host for r in seen] == ["proxy.example", "proxy.example"] assert seen[0].headers["Authorization"] == "Token sk-USERKEY" err = capsys.readouterr().err @@ -523,7 +533,7 @@ def test_ci_proxy_env_host_refused_for_agent( config = Config(config_path=write_user(tmp_path, KEYED)) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError) as err: - auth.guard() + send(auth) assert err.value.code == "foreign_host" assert seen == [] @@ -533,7 +543,7 @@ def test_cron_like_env_host_allowed_with_warning( monkeypatch.setenv("DEEPGRAM_BASE_URL", "https://selfhosted.example") config = Config(config_path=write_user(tmp_path, KEYED)) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen[0].headers["Authorization"] == "Token sk-USERKEY" assert capsys.readouterr().err.count("(from DEEPGRAM_BASE_URL)") == 1 @@ -544,7 +554,7 @@ def test_regional_or_allowlisted_env_host_no_warning( data = {**KEYED, "api": {"allowed_hosts": ["proxy.example"]}} config = Config(config_path=write_user(tmp_path, data)) auth, _ = recording_auth(config) - auth.guard() + send(auth) assert "(from DEEPGRAM_BASE_URL)" not in capsys.readouterr().err @@ -577,7 +587,7 @@ def test_default_path_widens_trust(self, tmp_path): assert config.get_profile().base_url == "http://127.0.0.1:8000" assert config.get("tools.ffprobe_path") == "/repo/ffprobe" auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen[0].url.host == "127.0.0.1" def test_other_path_cannot_widen_trust(self, tmp_path, agentic, monkeypatch): @@ -610,7 +620,7 @@ def test_other_path_cannot_widen_trust(self, tmp_path, agentic, monkeypatch): ) in texts assert not any(str(repo_file) in (h or "") for h in hints) auth, seen = recording_auth(config) - auth.guard() + send(auth) assert [r.url.host for r in seen] == ["api.deepgram.com"] def test_other_path_keeps_credentials_and_regional_url(self, tmp_path): @@ -633,7 +643,7 @@ def test_other_path_flag_host_still_foreign(self, tmp_path): ) auth, seen = recording_auth(config) with pytest.raises(CredentialHostError): - auth.guard() + send(auth) assert seen == [] @@ -717,7 +727,7 @@ def test_checked_url_is_the_url_used(self, tmp_path, flag): assert base == base.lower() assert base.startswith("https://api.") assert not base.endswith(".") - auth.guard() + send(auth) assert seen[0].url.host.startswith("api.") with ( patch.object(AuthManager, "guard"), @@ -764,7 +774,7 @@ def test_entry_forms(self, tmp_path, entry, recwarn): ) assert config.allowed_hosts() == ["proxy.example"] auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen assert not [w for w in recwarn if "serializer" in str(w.message).lower()] @@ -773,7 +783,7 @@ def test_entry_with_scheme_and_port(self, tmp_path): config = Config(config_path=write_user(tmp_path, data), base_url=LOOPBACK) assert config.allowed_hosts() == ["127.0.0.1:8000"] auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen def test_scheme_entry_does_not_match_host_named_https(self): @@ -791,7 +801,7 @@ def test_custom_config_reads_allowlist_from_default_file(tmp_path): config = Config(config_path=str(other), base_url="https://h.example") assert config.allowed_hosts() == ["h.example"] auth, seen = recording_auth(config) - auth.guard() + send(auth) assert seen[0].url.host == "h.example" @@ -802,6 +812,6 @@ def test_custom_config_hint_names_default_file(tmp_path, capsys): config = Config(config_path=str(other), base_url=FOREIGN) auth, _ = recording_auth(config) with pytest.raises(CredentialHostError) as err: - auth.guard() + send(auth) assert str(tmp_path / "user" / "config.yaml") in err.value.hint assert str(other) not in err.value.hint diff --git a/packages/deepctl-core/tests/unit/test_client.py b/packages/deepctl-core/tests/unit/test_client.py index 9b5d9174..17be91d2 100644 --- a/packages/deepctl-core/tests/unit/test_client.py +++ b/packages/deepctl-core/tests/unit/test_client.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, Mock, mock_open, patch import pytest +from deepctl_core import transport from deepctl_core.auth import AuthManager from deepctl_core.client import DeepgramClient from deepctl_core.config import Config @@ -128,6 +129,7 @@ def test_create_client_with_custom_base_url( # Verify DGClient was created with environment kwarg mock_dg_client.assert_called_once_with( api_key="sk-test", + httpx_client=transport.shared_client(), environment=mock_env.return_value, follow_redirects=False, ) diff --git a/packages/deepctl-core/tests/unit/test_transport.py b/packages/deepctl-core/tests/unit/test_transport.py new file mode 100644 index 00000000..b5ec530a --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_transport.py @@ -0,0 +1,729 @@ +"""The shared HTTP client and the catalogue's transport_mapping. + +Every row of the mapping goes through the real SDK facade +(``DeepgramClient``) on an ``httpx.MockTransport``, so the test sees what a +command sees. A loopback fake server checks the request and connection count. +""" + +from __future__ import annotations + +import json +import os +import socket +import ssl +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, ClassVar + +import click +import httpx +import pytest +import yaml +from click.testing import CliRunner +from deepctl_core import hosts, output, transport +from deepctl_core.auth import AuthManager +from deepctl_core.base_command import BaseCommand +from deepctl_core.client import DeepgramClient +from deepctl_core.config import Config +from deepctl_core.models import BaseResult +from deepctl_core.transport import TransportError +from deepctl_core.websocket import RedirectRefused + +KEY = "sk-W22SENTINELKEY" +_REAL_PROG = transport.prog + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "deepctl_core.config._default_config_file", + lambda: tmp_path / "user" / "config.yaml", + ) + monkeypatch.setattr( + "deepctl_core.config._trust_root_location", + lambda: (tmp_path, tmp_path / "user" / "config.yaml"), + ) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(transport, "_client", None) + monkeypatch.setattr(output, "is_agentic", lambda: False) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setitem(output._output_config, "agentic", False) + monkeypatch.setitem(output._output_config, "verbose", False) + monkeypatch.setitem(output._output_config, "quiet", False) + monkeypatch.setitem(output._output_config, "format", "default") + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setattr(transport, "prog", lambda: "dg") + from deepctl_core import auth as auth_module + + monkeypatch.setattr(auth_module.keyring, "get_password", lambda *a: None) + # The SDK retries 429/5xx and connect errors with a back-off sleep. + import deepgram.core.http_client as sdk_http + + monkeypatch.setattr(sdk_http.time, "sleep", lambda s: None) + monkeypatch.setenv("DEEPGRAM_API_KEY", KEY) + + +def facade(handler) -> tuple[DeepgramClient, list[httpx.Request]]: + seen: list[httpx.Request] = [] + + def record(request: httpx.Request) -> httpx.Response: + seen.append(request) + return handler(request) + + # Same redirect setting as transport.shared_client(). + transport._set_client( + httpx.Client(transport=httpx.MockTransport(record), follow_redirects=False) + ) + config = Config(config_path=str(Path.cwd() / "none.yaml")) + auth = AuthManager(config) + return DeepgramClient(config, auth), seen + + +def raising(exc_factory): + def handler(request: httpx.Request) -> httpx.Response: + raise exc_factory(request) + + return handler + + +def status(code: int, body: Any = None, headers: dict | None = None): + def handler(request: httpx.Request) -> httpx.Response: + if isinstance(body, (bytes, str)): + return httpx.Response(code, content=body, headers=headers) + return httpx.Response(code, json=body or {"err_msg": "x"}, headers=headers) + + return handler + + +def mapped(handler, call=lambda c: c.list_models()) -> TransportError: + client, _ = facade(handler) + with pytest.raises(TransportError) as err: + call(client) + return err.value + + +def _chained(outer, inner): + def make(request): + try: + raise inner + except BaseException as cause: + raise outer(str(cause), request=request) from cause + + return make + + +# ── transport_mapping, row by row ───────────────────────────────────── + + +@pytest.mark.parametrize( + ("code", "expected", "message"), + [ + (401, "auth_invalid", "Deepgram rejected the API key (HTTP 401)."), + ( + 403, + "auth_forbidden", + "This API key isn't allowed to make this request (HTTP 403).", + ), + (404, "not_found", "Deepgram returned 404 Not Found for /v1/models."), + (429, "rate_limited", "Deepgram is rate limiting requests (HTTP 429)."), + (400, "invalid_argument", "Deepgram rejected the request (HTTP 400): x"), + (422, "invalid_argument", "Deepgram rejected the request (HTTP 422): x"), + (418, "invalid_argument", "Deepgram rejected the request (HTTP 418): x"), + (500, "server_error", "Deepgram returned an error (HTTP 500)."), + (503, "server_error", "Deepgram returned an error (HTTP 503)."), + ], +) +def test_http_status_rows(code, expected, message): + err = mapped(status(code)) + assert err.code == expected + assert err.error == message + assert err.details["http_status"] == code + + +def test_429_retry_after_and_500_request_id(): + err = mapped(status(429, headers={"Retry-After": "7"})) + assert err.details == {"http_status": 429, "retry_after": 7} + err = mapped(status(503, headers={"dg-request-id": "req-123"})) + assert err.details == {"http_status": 503, "request_id": "req-123"} + + +def test_sdk_retries_before_mapping_5xx(): + """The SDK's own retries still run; the code comes from the last answer.""" + client, seen = facade(status(500)) + with pytest.raises(TransportError): + client.list_models() + assert len(seen) == 3 + + +def test_invalid_request_message_is_sanitized(): + body = { + "err_msg": f"bad \x1b[31mvalue {KEY} https://u:p@h.example/x?token=1 " + + "y" * 300 + } + err = mapped(status(400, body)) + assert KEY not in err.error and "\x1b" not in err.error + assert "u:p@" not in err.error and "token=1" not in err.error + api_message = err.values["api_message"] + assert len(api_message) <= 200 + + +def test_invalid_request_without_body_uses_reason_phrase(): + err = mapped(status(400, b"")) + assert err.error == "Deepgram rejected the request (HTTP 400): Bad Request" + + +@pytest.mark.parametrize( + ("factory", "reason"), + [ + ( + _chained( + httpx.ConnectError, socket.gaierror(-2, "Name or service not known") + ), + "name resolution failed", + ), + ( + _chained( + httpx.ConnectError, ConnectionRefusedError(111, "Connection refused") + ), + "connection refused", + ), + ( + _chained( + httpx.ConnectError, + ssl.SSLError(1, "[SSL: WRONG_VERSION_NUMBER] wrong version number"), + ), + "TLS handshake failed", + ), + ( + _chained( + httpx.ReadError, ConnectionResetError(104, "Connection reset by peer") + ), + "connection reset", + ), + ( + lambda r: httpx.RemoteProtocolError( + "peer closed connection without sending complete message body: incomplete chunked read", + request=r, + ), + "response cut off", + ), + ( + lambda r: httpx.RemoteProtocolError( + "Server disconnected without sending a response.", request=r + ), + "connection closed", + ), + ], +) +def test_network_rows(factory, reason): + err = mapped(raising(factory)) + assert err.code == "network_unavailable" + assert err.details == {"host": "api.deepgram.com", "reason": reason} + if reason == "response cut off": + assert err.messages == transport.NETWORK_CUT + assert err.error == ( + "The connection to api.deepgram.com dropped before the response finished." + ) + else: + assert err.messages == transport.CODES["network_unavailable"] + assert err.error == f"Couldn't reach api.deepgram.com: {reason}." + + +def test_unreadable_2xx_hint_names_host_and_prog(monkeypatch): + err = mapped(status(200, b"sign in")) + payload = _render(err, monkeypatch, agentic=False, hard=False) + assert payload["hint"] == transport.STRINGS["H-NETWORK-UNREADABLE"].format( + prog="dg", host="api.deepgram.com" + ) + + +def test_proxy_refusal_row(): + err = mapped(raising(lambda r: httpx.ProxyError("407 Proxy Auth", request=r))) + assert err.code == "network_unavailable" + assert err.details["reason"] == "the proxy refused the connection" + + +@pytest.mark.parametrize( + "exc", + [httpx.ReadTimeout, httpx.ConnectTimeout, httpx.WriteTimeout, httpx.PoolTimeout], +) +def test_timeout_row(exc): + err = mapped(raising(lambda r: exc("timed out", request=r))) + assert err.code == "timeout" + assert err.details == {"host": "api.deepgram.com", "timeout_s": 60} + assert err.error == "api.deepgram.com didn't respond within 60 s." + + +@pytest.mark.parametrize( + "body", [b'{"models": [{"name": "nov', b"captive portal"] +) +def test_unreadable_2xx_body_row(body): + """A 2xx whose body can't be read is a network failure, not success.""" + err = mapped(status(200, body)) + assert err.code == "network_unavailable" + assert err.details == {"host": "api.deepgram.com", "reason": "unreadable response"} + assert err.messages == transport.NETWORK_UNREADABLE + assert err.error == "Got a response from api.deepgram.com that dg couldn't read." + + +def test_speak_rest_maps_while_reading(): + """The speak request happens while the command iterates the audio.""" + client, _ = facade(status(401)) + chunks = client.speak_text("hi") + with pytest.raises(TransportError) as err: + list(chunks) + assert err.value.code == "auth_invalid" + + +def test_prerecorded_listen_maps(tmp_path): + audio = tmp_path / "a.wav" + audio.write_bytes(b"RIFF" + b"\0" * 64) + err = mapped(status(401), call=lambda c: c.transcribe_file(audio)) + assert err.code == "auth_invalid" + + +def test_non_transport_errors_keep_legacy_text(): + """A failure that isn't a request failure keeps its 0.4.0 wording.""" + from deepgram.core.api_error import ApiError + + client, seen = facade(status(200, {"projects": []})) + with pytest.raises(FileNotFoundError, match="Audio file not found"): + client.transcribe_file("/does/not/exist.wav") + client._resolve_project_id = lambda project_id=None: "p1" + + def broken(*args, **kwargs): + raise LookupError("sdk bug") + + client.client.manage.v1.models.list = broken + with pytest.raises(ApiError) as err: + client.list_models() + assert err.value.body == "Failed to list models: sdk bug" + assert seen == [] + + +def test_every_code_is_catalogued(): + for code, ids in [ + *transport.CODES.items(), + ("cut", transport.NETWORK_CUT), + ("unreadable", transport.NETWORK_UNREADABLE), + ]: + for message_id in ids: + if message_id: + assert message_id in transport.STRINGS, (code, message_id) + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_strings_match_catalogue(): + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + assert [row[1] for row in catalogue["transport_mapping"]] == [ + "network_unavailable", + "timeout", + "auth_invalid", + "auth_forbidden", + "not_found", + "rate_limited", + "invalid_argument", + "server_error", + ] + for message_id, text in transport.STRINGS.items(): + assert catalogue["strings"][message_id] == text, message_id + for code, (error_id, human, agent, automation) in transport.CODES.items(): + entry = catalogue["codes"][code] + assert entry["exit"] == 1 + message = next(m for m in entry["messages"] if m["id"] == error_id) + assert message["hint_human"] == human + assert message["hint_agent"] == agent + assert message.get("hint_automation") == automation + entry = catalogue["codes"]["network_unavailable"] + for extra, reason in ( + (transport.NETWORK_CUT, "response cut off"), + (transport.NETWORK_UNREADABLE, "unreadable response"), + ): + message = next(m for m in entry["messages"] if m["id"] == extra[0]) + assert (message["hint_human"], message["hint_agent"]) == extra[1:3] + assert "hint_automation" not in message + assert entry["details_values"][extra[0]] == {"reason": [reason]} + env = catalogue["codes"]["auth_invalid"]["env_key_variant"] + assert env == { + "hint_agent": "A-AUTH-INVALID-ENV", + "hint_agent_if_next_step_null": "A-AUTH-INVALID-ENV-NULL", + } + + +# ── Rendering: hint variant, next_step, key order ──────────────────── + + +def _render(err, monkeypatch, *, agentic, hard, command_path="models", key_env=True): + monkeypatch.setitem(output._output_config, "agentic", agentic) + monkeypatch.setattr(output, "is_ai_agent", lambda: hard) + if not key_env: + monkeypatch.delenv("DEEPGRAM_API_KEY") + config = Config(config_path=str(Path.cwd() / "none.yaml")) + auth = AuthManager(config, explicit_api_key=None if key_env else KEY) + return transport.render( + err, command_path=command_path, auth_manager=auth + ).model_dump() + + +def _401(): + return TransportError("auth_invalid", {}, {"http_status": 401}) + + +def test_auth_invalid_hard_agent_env_key(monkeypatch): + payload = _render(_401(), monkeypatch, agentic=True, hard=True) + assert list(payload) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + "details", + ] + assert payload == { + "error": "Deepgram rejected the API key (HTTP 401).", + "status": "error", + "code": "auth_invalid", + "hint": transport.STRINGS["A-AUTH-INVALID-ENV"], + "next_step": "dg login --yes", + "schema_version": 1, + "details": {"http_status": 401, "key_source": "DEEPGRAM_API_KEY (env)"}, + } + + +def test_auth_invalid_hard_agent_env_key_without_prog(monkeypatch): + monkeypatch.setattr(transport, "prog", lambda: None) + payload = _render(_401(), monkeypatch, agentic=True, hard=True) + assert payload["next_step"] is None + assert payload["hint"] == transport.STRINGS["A-AUTH-INVALID-ENV-NULL"].format( + prog="dg" + ) + + +def test_auth_invalid_soft_only_ci(monkeypatch): + payload = _render(_401(), monkeypatch, agentic=True, hard=False) + assert payload["next_step"] is None + assert payload["hint"] == transport.STRINGS["C-AUTH-INVALID"] + + +def test_auth_invalid_hard_agent_flag_key(monkeypatch): + payload = _render(_401(), monkeypatch, agentic=True, hard=True, key_env=False) + assert payload["next_step"] == "dg login" + assert payload["hint"] == transport.STRINGS["A-AUTH-INVALID"].format(prog="dg") + assert payload["details"]["key_source"] == "--api-key" + + +def test_auth_invalid_human(monkeypatch): + payload = _render(_401(), monkeypatch, agentic=False, hard=False) + assert payload["next_step"] == "dg login" + assert payload["hint"] == transport.STRINGS["H-AUTH-INVALID"].format(prog="dg") + + +def test_key_source_follows_whoami_order(monkeypatch): + """A keyring key equal to the env value is the keyring's, as whoami says.""" + from deepctl_core import auth as auth_module + + monkeypatch.setattr(auth_module.keyring, "get_password", lambda *a: KEY) + payload = _render(_401(), monkeypatch, agentic=True, hard=True) + assert payload["details"]["key_source"] == "system keyring" + assert payload["next_step"] == "dg login" + + +def test_invalid_argument_next_step_is_command_help(monkeypatch): + err = TransportError( + "invalid_argument", + {"http_status": 400, "api_message": "x"}, + {"http_status": 400}, + ) + payload = _render(err, monkeypatch, agentic=True, hard=True, command_path="keys") + assert payload["next_step"] == "dg keys --help" + assert payload["hint"] == transport.STRINGS["A-INVALID-REQUEST"].format( + prog="dg", command_path="keys" + ) + + +@pytest.mark.parametrize( + "code", + [ + "auth_forbidden", + "not_found", + "rate_limited", + "server_error", + "timeout", + "network_unavailable", + ], +) +def test_other_codes_have_null_next_step(code, monkeypatch): + values = { + "host": "h", + "reason": "r", + "timeout_s": 60, + "path": "/p", + "http_status": 500, + } + err = TransportError(code, values, {"http_status": 1}) + payload = _render(err, monkeypatch, agentic=True, hard=True) + assert payload["next_step"] is None and payload["schema_version"] == 1 + + +def test_details_omitted_when_empty(): + cls = transport.result_class() + payload = cls(error="e", code="c", hint="h").model_dump() + assert list(payload) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + ] + + +def test_prog_bare_name_only_when_which_matches(monkeypatch, tmp_path): + script = tmp_path / "bin" / "dg" + script.parent.mkdir() + script.write_text("#!/bin/sh\n") + script.chmod(0o755) + monkeypatch.setattr(sys, "argv", [str(script)]) + monkeypatch.setenv("PATH", str(script.parent)) + assert _REAL_PROG() == "dg" + monkeypatch.setenv("PATH", "/nonexistent") + assert _REAL_PROG() == str(script) + monkeypatch.setattr(sys, "argv", ["/x/__main__.py"]) + assert _REAL_PROG() is None + + +# ── Through BaseCommand: stdout payload, two stderr lines, exit 1 ───── + + +class _Models(BaseCommand): + name = "models" + help = "List models" + requires_auth = True + + def handle(self, config, auth_manager, client, **kwargs): + from deepctl_core.output import get_status_console + + get_status_console().print("[blue]Fetching models...[/blue]") + try: + client.list_models() + except Exception as e: # what every command does today + get_status_console().print(f"[red]Error:[/red] {e}") + return BaseResult(status="error", message=str(e)) + return BaseResult(status="success") + + +def _run( + monkeypatch, + handler, + *, + agentic, + hard, + fmt="json", + config_path="none.yaml", + name="models", +): + facade(handler) + monkeypatch.setitem(output._output_config, "agentic", agentic) + monkeypatch.setitem(output._output_config, "format", fmt) + monkeypatch.setattr(output, "is_ai_agent", lambda: hard) + cmd = type("_Cmd", (_Models,), {"name": name})() + + @click.command(name) + @click.pass_context + def models(ctx): + cmd.execute(ctx) + + @click.group("dg") + @click.pass_context + def root(ctx): + ctx.obj = {"config": Config(config_path=str(Path.cwd() / config_path))} + + root.add_command(models) + return CliRunner().invoke(root, [name]) + + +def _split(result): + return result.stdout, result.stderr + + +def test_mode_a_two_stderr_lines_and_coded_stdout(monkeypatch): + monkeypatch.delenv("DEEPGRAM_API_KEY") + Path("user").mkdir() + Path("user/config.yaml").write_text( + yaml.safe_dump({"profiles": {"default": {"api_key": KEY}}}) + ) + Path("user/config.yaml").chmod(0o600) + result = _run( + monkeypatch, + status(401), + agentic=True, + hard=True, + config_path="user/config.yaml", + ) + stdout, stderr = _split(result) + assert result.exit_code == 1 + payload = json.loads(stdout) + assert payload["code"] == "auth_invalid" + assert payload["next_step"] == "dg login" + assert payload["details"] == {"http_status": 401, "key_source": "config file"} + assert stderr.splitlines()[-2:] == [ + "ERROR: Deepgram rejected the API key (HTTP 401).", + "HINT: " + transport.STRINGS["A-AUTH-INVALID"].format(prog="dg"), + ] + assert KEY not in stdout + stderr + + +def test_human_mode_renders_cross_and_hint(monkeypatch): + result = _run(monkeypatch, status(403), agentic=False, hard=False, fmt="default") + stdout, stderr = _split(result) + assert result.exit_code == 1 + assert stdout == "" + lines = [line for line in stderr.splitlines() if line.strip()] + assert lines[-2:] == [ + "✗ This API key isn't allowed to make this request (HTTP 403).", + " " + transport.STRINGS["H-AUTH-FORBIDDEN"].format(prog="dg"), + ] + + +def test_failure_after_stdout_payload_keeps_json_off_stdout(monkeypatch): + """Once a command has written payload bytes to stdout (speak audio), a + failure goes to stderr only.""" + + class _Audio(_Models): + def handle(self, config, auth_manager, client, **kwargs): + import sys + + self.stdout_is_protocol = True + sys.stdout.write("AUDIOBYTES") + client.list_models() + + monkeypatch.setattr(sys.modules[__name__], "_Models", _Audio) + result = _run(monkeypatch, status(200, b"cut"), agentic=True, hard=True) + assert result.exit_code == 1 + assert result.stdout == "AUDIOBYTES" + assert result.stderr.splitlines()[-2].startswith("ERROR: Got a response from") + + +def test_unmapped_failure_keeps_legacy_handling(monkeypatch): + """Errors that aren't request failures still go through the command's + own handler, prefixed lines intact.""" + + def boom(request): + raise ValueError("not a transport problem") + + result = _run(monkeypatch, boom, agentic=True, hard=False) + assert result.exit_code == 1 + assert "Error:" in result.stderr + assert json.loads(result.stdout)["status"] == "error" + + +# ── One connection, no pre-flight: a real loopback server ───────────── + + +class _Fake(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + log: ClassVar[list[tuple[str, str, int]]] = [] + + def do_GET(self): + _Fake.log.append( + (self.command, self.path.split("?")[0], self.client_address[1]) + ) + body = json.dumps({"api_keys": [], "projects": [], "models": []}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +@pytest.fixture +def fake_server(): + _Fake.log = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _Fake) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_address[1]}" + server.shutdown() + server.server_close() + + +def test_one_request_one_connection_no_preflight(fake_server, monkeypatch): + monkeypatch.setenv("DEEPGRAM_BASE_URL", fake_server) + monkeypatch.setenv("DEEPGRAM_PROJECT_ID", "p1") + config = Config(config_path=str(Path.cwd() / "none.yaml")) + auth = AuthManager(config) + auth.guard() + client = DeepgramClient(config, auth) + client.list_keys() + client.list_models() + paths = [p for _, p, _ in _Fake.log] + assert paths == ["/v1/projects/p1/keys", "/v1/models"] + assert "/v1/projects" not in paths + assert len({port for *_, port in _Fake.log}) == 1 + + +def test_shared_client_keeps_no_cookies(): + seen: list[httpx.Request] = [] + + def handler(request): + seen.append(request) + return httpx.Response( + 200, json={"models": []}, headers={"Set-Cookie": "sid=abc; Path=/"} + ) + + client, _ = facade(handler) + client.list_models() + client.list_models() + assert "cookie" not in seen[1].headers + assert not list(transport._client.cookies.jar) + + +def test_shared_client_settings(): + client = transport.shared_client() + assert client.follow_redirects is False + assert client.timeout.read == transport.DEFAULT_TIMEOUT_S + assert "Authorization" not in client.headers + assert str(client.base_url) == "" + + +def test_redirect_to_another_origin_is_not_followed(): + """The shared client follows no redirect, so neither the key nor a + request body can reach another origin.""" + seen: list[httpx.Request] = [] + + def handler(request): + seen.append(request) + if request.url.host == "api.deepgram.com": + return httpx.Response( + 302, headers={"Location": "https://evil.example/v1/models"} + ) + return httpx.Response(200, json={"models": []}) + + client, _ = facade(handler) + with pytest.raises(RedirectRefused): + client.list_models() + assert [r.url.host for r in seen] == ["api.deepgram.com"] + assert seen[0].headers["Authorization"] == f"Token {KEY}" + + +def test_shared_client_is_lazy_and_single(): + code = ( + "import sys, deepctl_core.transport as t\n" + "print('httpx' in sys.modules, 'ssl' in sys.modules)\n" + "assert t.shared_client() is t.shared_client()\n" + ) + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.strip() == "False False" From 648576e36a42919edad009e8f8f9be8332378556 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 11:30:01 -0400 Subject: [PATCH 04/18] fix(output): drop status chrome from stderr in agent mode In mode A every stderr line carries a prefix and status chrome is suppressed. The commands that use the shared client (billing, keys, listen, members, models, projects, read, requests, speak, transcribe, usage) now run their handler with stderr filtered: only ERROR:/WARN:/INFO:/HINT:/OK:/DEPRECATED: lines, and 0.4.0 "Error"/"Warning" lines, get through. Each write is kept or dropped as a whole. --verbose keeps everything, and every other command (login's device code, dg api, debug-probe) is unchanged. Lines in those commands that a caller needs now render as warnings or errors instead of chrome, so they survive the filter: listen's Flux --diarize note, its --encoding assumption and the microphone audio warning; speak's --play note; projects' "Failed to ..." errors and the missing project_id warning. The listen and speak warnings print under --quiet too (print_warning gains ignore_quiet), since --quiet hides only INFO and OK lines. --- .../src/deepctl_cmd_listen/command.py | 24 +++-- .../tests/unit/test_listen_command.py | 55 ++++++++++-- .../src/deepctl_cmd_projects/command.py | 20 +++-- .../tests/unit/test_projects_command.py | 22 +++++ .../src/deepctl_cmd_speak/command.py | 9 +- .../src/deepctl_core/base_command.py | 90 ++++++++++++++++++- .../deepctl-core/src/deepctl_core/output.py | 13 ++- .../deepctl-core/tests/unit/test_transport.py | 33 ++++++- 8 files changed, 235 insertions(+), 31 deletions(-) diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index fac2995c..41ff9f12 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -28,9 +28,10 @@ Config, DeepgramClient, ) -from deepctl_core.output import _agentic +from deepctl_core.output import _agentic, print_warning from deepctl_core.websocket import connect_async from rich.console import Console +from rich.markup import escape from rich.prompt import Confirm, Prompt from rich.table import Table @@ -402,9 +403,10 @@ def handle( # from an explicit flag, so they stay silent; --interim still gates # client-side display.) if api_version >= 2 and diarize: - status.print( - "[yellow]Note:[/yellow] --diarize is not supported by Flux STT " - "(listen v2) models; ignoring it." + print_warning( + "Flux models don't support --diarize, so it was ignored.", + stderr=True, + ignore_quiet=True, ) diarize = False save_to = kwargs.get("save_to") @@ -888,7 +890,11 @@ async def send_audio() -> None: def callback(indata: Any, frames: int, t: Any, s: Any) -> None: if s: - status.print(f"[yellow]Audio warning: {s}[/yellow]") + print_warning( + f"Audio warning: {escape(str(s))}", + stderr=True, + ignore_quiet=True, + ) loop.call_soon_threadsafe(q.put_nowait, bytes(indata)) stream = sd.RawInputStream( @@ -972,9 +978,11 @@ def _stream_stdin( ) -> BaseResult: resolved_encoding = encoding or "linear16" if not encoding: - status.print( - "[dim]No --encoding specified — assuming linear16. " - "Pass --encoding to suppress this warning.[/dim]" + print_warning( + "No --encoding specified — assuming linear16. " + "Pass --encoding to suppress this warning.", + stderr=True, + ignore_quiet=True, ) caption_writer: StreamingCaptionWriter | None = None diff --git a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py index 6b92fe2d..c0fecbfb 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py @@ -229,12 +229,14 @@ def test_handle_stdin_routes_to_stream_stdin( assert call_kwargs["encoding"] == "linear16" assert result.source == "stdin" + @patch("deepctl_cmd_listen.command.print_warning") @patch("deepctl_cmd_listen.command.status") @patch("deepctl_cmd_listen.command.sys") def test_handle_warns_diarize_ignored_on_flux( self, mock_sys, mock_status, + mock_warning, command, mock_config, mock_auth_manager, @@ -261,20 +263,57 @@ async def fake_ws_mic(_client, **kwargs): diarize=True, ) - printed = " ".join( - str(c.args[0]) for c in mock_status.print.call_args_list if c.args - ) - assert "not supported by Flux STT" in printed + # The note is a warning (WARN: in agent mode), not status chrome. + calls = mock_status.print.call_args_list + mock_warning.call_args_list + printed = " ".join(str(c.args[0]) for c in calls if c.args) + assert "Flux models don't support --diarize" in printed assert "Speaker labels enabled" not in printed assert mock_ws.call_args.kwargs["diarize"] is False assert result.diarized is False + @patch("deepctl_cmd_listen.command.status") + @patch("deepctl_cmd_listen.command.sys") + def test_diarize_warning_prints_under_quiet( + self, + mock_sys, + mock_status, + command, + mock_config, + mock_auth_manager, + mock_client, + monkeypatch, + capsys, + ): + """`dg -q listen ... --diarize`: --quiet hides INFO/OK lines, not this + WARN.""" + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "quiet", True) + monkeypatch.setitem(output._output_config, "agentic", True) + mock_sys.stdin.isatty.return_value = True + + async def fake_ws_mic(_client, **kwargs): + return ListenResult(status="success", source="mic", mode="live") + + with patch.object(command, "_ws_mic", side_effect=fake_ws_mic): + command.handle( + config=mock_config, + auth_manager=mock_auth_manager, + client=mock_client, + mic=True, + model="flux-general-en", + diarize=True, + ) + assert "WARN: Flux models don't support --diarize, so it was ignored." in capsys.readouterr().err + + @patch("deepctl_cmd_listen.command.print_warning") @patch("deepctl_cmd_listen.command.status") @patch("deepctl_cmd_listen.command.sys") def test_handle_no_diarize_warning_on_v1( self, mock_sys, mock_status, + mock_warning, command, mock_config, mock_auth_manager, @@ -294,10 +333,10 @@ def test_handle_no_diarize_warning_on_v1( diarize=True, ) - printed = " ".join( - str(c.args[0]) for c in mock_status.print.call_args_list if c.args - ) - assert "not supported by Flux STT" not in printed + # The note is a warning (WARN: in agent mode), not status chrome. + calls = mock_status.print.call_args_list + mock_warning.call_args_list + printed = " ".join(str(c.args[0]) for c in calls if c.args) + assert "don't support --diarize" not in printed @pytest.mark.parametrize("source", ["call.wav", "https://example.com/call.wav"]) def test_click_flux_prerecorded_source_errors(self, source, command, mock_config): diff --git a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py index b34b149c..cd2a6896 100644 --- a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py +++ b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py @@ -11,8 +11,10 @@ get_output_format, get_status_console, ) +from deepctl_core.output import print_error, print_warning from deepctl_core.transport import TransportError from rich.console import Console +from rich.markup import escape from .models import ProjectInfo, ProjectsResult @@ -184,13 +186,15 @@ def _list_projects(self, client: DeepgramClient) -> ProjectsResult | BaseResult: ) except Exception as e: - status_console.print(f"[red]Failed to list projects:[/red] {e}") + print_error(f"Failed to list projects: {escape(str(e))}") return BaseResult(status="error", message=str(e)) def _create_project( self, client: DeepgramClient, name: str, company: str | None ) -> ProjectsResult | BaseResult: """Create a new project.""" + from deepctl_core.transport import prog + status_console.print(f"[blue]Creating project:[/blue] {name}") if company: @@ -221,8 +225,12 @@ def _create_project( count=1, ) else: - status_console.print( - "[yellow]Project creation response missing project_id[/yellow]" + print_warning( + escape( + "Deepgram created the project but didn't return its " + f"ID. Run {prog() or 'dg'} projects --list to find it." + ), + stderr=True, ) return ProjectsResult( status="warning", @@ -232,7 +240,7 @@ def _create_project( ) except Exception as e: - status_console.print(f"[red]Failed to create project:[/red] {e}") + print_error(f"Failed to create project: {escape(str(e))}") return BaseResult(status="error", message=str(e)) def _show_project( @@ -287,7 +295,7 @@ def _show_project( ) except Exception as e: - status_console.print(f"[red]Failed to get project details:[/red] {e}") + print_error(f"Failed to get project details: {escape(str(e))}") return BaseResult(status="error", message=str(e)) def _show_current_project( @@ -349,7 +357,7 @@ def _set_default_project( ) except Exception as e: - status_console.print(f"[red]Failed to set default project:[/red] {e}") + print_error(f"Failed to set default project: {escape(str(e))}") from deepctl_core.output import print_error_hint print_error_hint(e) diff --git a/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py b/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py index 67b86260..7041e02f 100644 --- a/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py +++ b/packages/deepctl-cmd-projects/tests/unit/test_projects_command.py @@ -106,3 +106,25 @@ def test_other_failures_stop(self): with pytest.raises(TransportError): self._run("server_error") + + +def test_create_without_an_id_says_how_to_find_it(monkeypatch): + monkeypatch.setattr("deepctl_core.transport.prog", lambda: "dg") + warnings: list[str] = [] + monkeypatch.setattr( + "deepctl_cmd_projects.command.print_warning", + lambda text, **kwargs: warnings.append(text), + ) + client = Mock(spec=DeepgramClient) + client.create_project.return_value = {"name": "Acme"} + result = ProjectsCommand().handle( + config=Mock(spec=Config), + auth_manager=Mock(spec=AuthManager), + client=client, + create="Acme", + ) + assert result.status == "warning" + assert warnings == [ + "Deepgram created the project but didn't return its ID. Run dg projects " + "--list to find it." + ] diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index df76e6b7..3602cb62 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -24,6 +24,7 @@ get_output_format, get_status_console, ) +from deepctl_core.output import print_warning from rich.console import Console from rich.markup import escape from rich.table import Table @@ -600,9 +601,11 @@ def handle( # Playing sends the audio to the player, not to stdout, so a # redirect or pipe would otherwise collect nothing and give no # hint why. - console.print( - "[yellow]Note:[/yellow] --play sends the audio to your player, " - "so stdout stays empty. Add -o to save a file too." + print_warning( + "--play sends the audio to your player, so stdout stays empty. " + "Add -o to save a file too.", + stderr=True, + ignore_quiet=True, ) if is_flux: diff --git a/packages/deepctl-core/src/deepctl_core/base_command.py b/packages/deepctl-core/src/deepctl_core/base_command.py index 182a198d..1b930b31 100644 --- a/packages/deepctl-core/src/deepctl_core/base_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_command.py @@ -1,7 +1,8 @@ """Base command class for deepctl commands.""" +import sys from abc import ABC, abstractmethod -from typing import Any, ClassVar +from typing import Any, ClassVar, TextIO import click @@ -12,6 +13,7 @@ from .models import CodedErrorResult, ErrorResult from .output import ( _agentic, + _output_config, print_error, print_error_hint, print_hint, @@ -28,6 +30,69 @@ console = stdout_console +# Mode A stderr grammar: every line starts with one of +# these. Lines that start with "Error"/"Warning" are 0.4.0 error text a +# command printed itself, kept until a later release codes them. +_MODE_A_PREFIXES = ( + "ERROR: ", + "WARN: ", + "INFO: ", + "HINT: ", + "OK: ", + "DEPRECATED: ", + "Error", + "Warning", +) + +# Built-in commands whose requests go through DeepgramClient and the shared +# Transport. In mode A their handlers run without status chrome on +# stderr. Other commands print things the caller needs there (login's device +# code, dg api's response errors, debug-probe's proxy address), so they keep +# it. Built-in names can't be taken by plugins. +_CHROME_FREE_COMMANDS = frozenset( + { + "billing", + "keys", + "listen", + "members", + "models", + "projects", + "read", + "requests", + "speak", + "transcribe", + "usage", + } +) + + +class _ModeAStderr: + """``sys.stderr`` for a command's handler in mode A: drops status chrome. + + Each write that starts a line is kept or dropped as a whole, so one + ``print`` that wraps over several lines stays together. + """ + + def __init__(self, stream: TextIO) -> None: + self._stream = stream + self._keep = True + self._line_start = True + + def write(self, text: str) -> int: + if not text: + return 0 + if self._line_start: + first = text.lstrip("\n").split("\n", 1)[0] + self._keep = bool(first.strip()) and first.startswith(_MODE_A_PREFIXES) + if self._keep: + self._stream.write(text) + self._line_start = text.endswith("\n") + return len(text) + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + class UnexpectedCommandError(click.ClickException): """A command failed with an error it didn't raise on purpose. @@ -196,7 +261,7 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: # Execute the command try: with TimingContext(f"command_{self.name}_handler"): - result = self.handle(config, auth_manager, client, **kwargs) + result = self._run_handler(config, auth_manager, client, **kwargs) # Handle command result if result is not None: @@ -245,6 +310,27 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: if exit_code: raise SystemExit(exit_code) + def _run_handler( + self, + config: Config, + auth_manager: AuthManager, + client: DeepgramClient, + **kwargs: Any, + ) -> Any: + """Run ``handle``; in mode A, without status chrome on stderr.""" + if ( + not _output_config["agentic"] + or _output_config["verbose"] + or self.name not in _CHROME_FREE_COMMANDS + ): + return self.handle(config, auth_manager, client, **kwargs) + real = sys.stderr + sys.stderr = _ModeAStderr(real) + try: + return self.handle(config, auth_manager, client, **kwargs) + finally: + sys.stderr = real + def _report_transport_error( self, error: TransportError, diff --git a/packages/deepctl-core/src/deepctl_core/output.py b/packages/deepctl-core/src/deepctl_core/output.py index f05d231a..b7270631 100644 --- a/packages/deepctl-core/src/deepctl_core/output.py +++ b/packages/deepctl-core/src/deepctl_core/output.py @@ -502,9 +502,16 @@ def print_error_hint(error: BaseException) -> None: print_hint(str(hint)) -def print_warning(message: str, *, stderr: bool = False) -> None: - """Print a warning; ``stderr=True`` keeps it off stdout outside agentic mode too.""" - if not _output_config["quiet"]: +def print_warning( + message: str, *, stderr: bool = False, ignore_quiet: bool = False +) -> None: + """Print a warning; ``stderr=True`` keeps it off stdout outside agentic mode too. + + ``ignore_quiet=True`` prints it under ``--quiet`` as well, for warnings + the caller needs to read the result (``--quiet`` hides only + INFO and OK lines). + """ + if ignore_quiet or not _output_config["quiet"]: if _output_config["agentic"]: stderr_console.print(f"WARN: {message}", soft_wrap=True) else: diff --git a/packages/deepctl-core/tests/unit/test_transport.py b/packages/deepctl-core/tests/unit/test_transport.py index b5ec530a..3b980e6c 100644 --- a/packages/deepctl-core/tests/unit/test_transport.py +++ b/packages/deepctl-core/tests/unit/test_transport.py @@ -574,7 +574,7 @@ def test_mode_a_two_stderr_lines_and_coded_stdout(monkeypatch): assert payload["code"] == "auth_invalid" assert payload["next_step"] == "dg login" assert payload["details"] == {"http_status": 401, "key_source": "config file"} - assert stderr.splitlines()[-2:] == [ + assert stderr.splitlines() == [ "ERROR: Deepgram rejected the API key (HTTP 401).", "HINT: " + transport.STRINGS["A-AUTH-INVALID"].format(prog="dg"), ] @@ -593,6 +593,29 @@ def test_human_mode_renders_cross_and_hint(monkeypatch): ] +def test_success_in_mode_a_drops_chrome_only(monkeypatch): + result = _run(monkeypatch, status(200, {"models": []}), agentic=True, hard=False) + stdout, stderr = _split(result) + assert result.exit_code == 0 + assert "Fetching models" not in stderr + assert json.loads(stdout)["status"] == "success" + + +def test_other_commands_keep_chrome_in_mode_a(monkeypatch): + """Only commands that go through the Transport drop chrome: login's + device code, dg api's errors and the like stay on stderr.""" + result = _run( + monkeypatch, status(200, {"models": []}), agentic=True, hard=True, name="login" + ) + assert "Fetching models" in result.stderr + + +def test_verbose_keeps_chrome(monkeypatch): + monkeypatch.setitem(output._output_config, "verbose", True) + result = _run(monkeypatch, status(200, {"models": []}), agentic=True, hard=False) + assert "Fetching models" in result.stderr + + def test_failure_after_stdout_payload_keeps_json_off_stdout(monkeypatch): """Once a command has written payload bytes to stdout (speak audio), a failure goes to stderr only.""" @@ -727,3 +750,11 @@ def test_shared_client_is_lazy_and_single(): [sys.executable, "-c", code], capture_output=True, text=True, check=True ) assert out.stdout.strip() == "False False" + + +@pytest.mark.parametrize("ignore_quiet", [False, True]) +def test_print_warning_ignore_quiet(ignore_quiet, monkeypatch, capsys): + monkeypatch.setitem(output._output_config, "quiet", True) + monkeypatch.setitem(output._output_config, "agentic", True) + output.print_warning("needed", stderr=True, ignore_quiet=ignore_quiet) + assert ("WARN: needed" in capsys.readouterr().err) is ignore_quiet From 9a8a9c2eaca568834cd941ef250696e6587af850 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:30:11 -0400 Subject: [PATCH 05/18] fix(listen): finish live streams cleanly on Ctrl-C and end of input Live stdin and microphone streams run through one session engine (deepctl_cmd_listen.live) with defined endings: - Ctrl-C (Ctrl-Break on Windows): send CloseStream, keep receiving for at most 1 s, keep the finals and --save-to, exit 2. On --mic, Ctrl-C is how the session ends: the same drain, then exit 0 with the transcript so far, also when it comes before the session has connected. Ctrl-Break stays a hard stop (exit 2) on --mic. A second Ctrl-C exits at once. The handler is signal.signal, not add_signal_handler, so it works on every platform and on Python 3.10. - SIGTERM on POSIX (docker stop, systemd, Kubernetes): the same drain, then the process re-raises SIGTERM with the default action, so the wait status stays "killed by signal 15" (143 in a shell). A second signal ends it at once. - End of input: send CloseStream and wait for the server's close under an idle deadline (10 s since the last server message) and an absolute cap (max(30 s, 2 x the audio still behind for PCM/mulaw/alaw, or 2 x the time spent sending otherwise)). If either expires the run ends with stream_drain_timeout, exit 1. Ctrl-C during the drain switches to the 1 s cap and exit 2. - A signal that arrives while the server is closing still counts: the run ends as stopped by that signal. - The server closes: exit 0, or exit 1 with server_error when it reported an error. - The connection drops: exit 1 with network_unavailable. - Whoever reads stdout goes away (`| head -n 1`, `| true`): the stream ends quietly and exits 0. Captions are flushed line by line, so they also arrive as they are produced. - stdin can't be read (a write-only or broken descriptor, EIO on a hang-up): exit 1 with invalid_argument and a hint to pass a file or URL instead. Each failure is a coded error whose details carry the finals received so far (transcript, transcript_chars, saved_to), so nothing that arrived is lost. Text the server sends back (close reasons, Flux error descriptions) is stripped of control characters and of the API key and any Token/Bearer value before it is shown or saved. Stdin is read with os.read in a daemon thread that hands each buffer to the loop through run_coroutine_threadsafe and waits for it to be taken. The thread checks a closing event before every read and hand-off, shutdown sets that event before it cancels any task, and anything raised in the thread ends it quietly. This removes the interpreter abort (exit 134, "Fatal Python error") that a reader blocked in sys.stdin.buffer caused at shutdown. The reader blocks SIGINT and SIGTERM so they always reach the main thread, and waits on a stdin left non-blocking by its parent instead of failing. Regular files, pipes, /dev/null and TTYs all work. At interpreter exit the remaining objects are frozen out of the garbage collector, so the process ends a few milliseconds after the server's close instead of about 100 ms later. The session opens through deepctl_core.websocket.open_async, the same no-redirect open as connect_async but returning the connection and taking websockets options, so a 3xx on the upgrade is still refused before a second connection (mic and stdin), as in 0.4.1. deepctl-core now declares websockets>=13.1,<18, which that module imports. --- .../src/deepctl_cmd_listen/captions.py | 26 +- .../src/deepctl_cmd_listen/command.py | 634 ++++++++------ .../src/deepctl_cmd_listen/live.py | 673 +++++++++++++++ .../src/deepctl_cmd_listen/messages.py | 72 ++ .../src/deepctl_cmd_listen/models.py | 19 +- .../tests/unit/test_listen_command.py | 320 ++++--- .../tests/unit/test_live.py | 780 ++++++++++++++++++ .../tests/unit/test_live_command.py | 241 ++++++ .../tests/unit/test_live_exit.py | 152 ++++ packages/deepctl-core/pyproject.toml | 1 + .../src/deepctl_core/websocket.py | 93 ++- tests/e2e/conftest.py | 14 +- uv.lock | 3 + 13 files changed, 2560 insertions(+), 468 deletions(-) create mode 100644 packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py create mode 100644 packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_live.py create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_live_command.py create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_live_exit.py diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py index 479225eb..355257a5 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py @@ -107,12 +107,12 @@ def print_header(self, note: str | None = None) -> None: etc.) in a spec-compliant way. SRT has no header equivalent. """ if self.fmt == "webvtt": - print("WEBVTT") - print() + print("WEBVTT", flush=True) + print(flush=True) if note: - print("NOTE") - print(note) - print() + print("NOTE", flush=True) + print(note, flush=True) + print(flush=True) def write_entry( self, @@ -137,17 +137,17 @@ def write_entry( first_speaker: int | None = words[0].get("speaker") if words else None if self.fmt == "webvtt": - print(f"{_fmt_webvtt(start)} --> {_fmt_webvtt(end)}") + print(f"{_fmt_webvtt(start)} --> {_fmt_webvtt(end)}", flush=True) prefix = f"" if first_speaker is not None else "" - print(f"{prefix}{text}") - print() + print(f"{prefix}{text}", flush=True) + print(flush=True) else: - print(self._count) - print(f"{_fmt_srt(start)} --> {_fmt_srt(end)}") + print(self._count, flush=True) + print(f"{_fmt_srt(start)} --> {_fmt_srt(end)}", flush=True) if first_speaker is not None: - print(f"[Speaker {first_speaker}]") - print(text) - print() + print(f"[Speaker {first_speaker}]", flush=True) + print(text, flush=True) + print(flush=True) @property def accumulated_words(self) -> list[dict[str, Any]]: diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 41ff9f12..252507d7 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -14,11 +14,13 @@ from __future__ import annotations import json +import os +import signal import sys import threading from pathlib import Path from typing import TYPE_CHECKING, Any -from urllib.parse import urlencode +from urllib.parse import urlencode, urlsplit import click from deepctl_core import ( @@ -28,13 +30,20 @@ Config, DeepgramClient, ) -from deepctl_core.output import _agentic, print_warning -from deepctl_core.websocket import connect_async +from deepctl_core.hosts import CredentialHostError +from deepctl_core.output import ( + _agentic, + is_ai_agent, + print_error, + print_hint, + print_warning, +) from rich.console import Console from rich.markup import escape from rich.prompt import Confirm, Prompt from rich.table import Table +from . import messages as msg from .captions import ( StreamingCaptionWriter, captions_from_prerecorded, @@ -48,11 +57,13 @@ format_diarized_transcript, format_diarized_words, ) -from .models import ListenResult +from .models import ListenCodedError, ListenResult if TYPE_CHECKING: import asyncio + from . import live + # Status messages → stderr so stdout stays clean for transcript piping status = Console(stderr=True) # Structured output (table, json) → stdout @@ -80,41 +91,62 @@ def _ws_base(client: DeepgramClient) -> str: return f"{'ws' if scheme.lower() in ('http', 'ws') else 'wss'}://{rest}" -def _live_connect(url: str, api_key: str | None) -> Any: - """Open the live WebSocket for ``async with``, without redirects. - - A 3xx on the upgrade, same origin or not, stops the command before a - second connection is opened, so the key and the audio go only to the - host that passed the host check. - """ - return connect_async( - url, - {"Authorization": f"Token {api_key}"}, - command="dg listen", - payload="audio", - ) - - # asyncio (and with it ssl) loads only when a live stream starts, so # `dg listen --help` and the prerecorded path don't import it. +# The live engine imports asyncio, so it loads lazily too. def __getattr__(name: str) -> Any: if name == "asyncio": import asyncio return asyncio + if name == "live": + from . import live + + return live raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -async def _cancel_and_drain(*tasks: asyncio.Task[Any]) -> None: - """Cancel tasks and consume their terminal exceptions during teardown.""" - for task in tasks: - if not task.done(): - task.cancel() - for task in tasks: - try: - await task - except BaseException: - pass +_freeze_registered = False + + +def _freeze_at_exit() -> None: + """At interpreter exit, move every live object out of the collector's + reach. The final collection then takes a few ms instead of about 100, + so dg exits right after the server's close (plan T23). Registered with + atexit, so a long-lived host that runs the command is unaffected.""" + global _freeze_registered + if _freeze_registered: + return + _freeze_registered = True + import atexit + import gc + + atexit.register(gc.freeze) + + +def _stdin_fd() -> int: + """The descriptor live stdin streaming reads (a seam for tests).""" + return 0 + + +def _prog() -> str: + """The program name to show in messages and ``next_step``. + + The bare name when it resolves on PATH to the running executable; + otherwise ``dg``. + """ + import shutil + + try: + argv0 = sys.argv[0] + name = os.path.basename(argv0) + if name in ("dg", "deepgram", "deepctl"): + found = shutil.which(name) + if found and os.path.realpath(found) == os.path.realpath(argv0): + return name + except Exception: + pass + return "dg" class ListenCommand(BaseCommand): @@ -133,6 +165,11 @@ class ListenCommand(BaseCommand): requires_project = False ci_friendly = True + #: Host of the live socket, for network_unavailable details. + _ws_host: str = "" + #: Set after a SIGTERM drain; exit_code_for re-raises it. + _reraise_signal: int | None = None + examples = [ "dg listen audio.mp3", "dg listen https://example.com/call.mp3 --diarize", @@ -330,7 +367,7 @@ def handle( auth_manager: AuthManager, client: DeepgramClient, **kwargs: Any, - ) -> BaseResult: + ) -> BaseResult | ListenCodedError: source: str | None = kwargs.get("source") use_mic: bool = kwargs.get("mic", False) guided_flow = False @@ -747,7 +784,7 @@ def _prerecorded( duration_seconds=duration_seconds, ) - # ── Live mic handler ─────────────────────────────────────────────── + # ── Live streaming (mic and stdin) ───────────────────────────────── def _stream_mic( self, @@ -766,7 +803,15 @@ def _stream_mic( channels: int, save_to: str | None, caption_format: str | None, - ) -> BaseResult: + ) -> BaseResult | ListenCodedError: + import asyncio + + from . import live + + # From here on Ctrl-C ends the mic session, even before the + # session's own handler is in place (imports, connecting). + early = live.EarlyStop() + early.install() caption_writer: StreamingCaptionWriter | None = None if caption_format: @@ -775,9 +820,12 @@ def _stream_mic( if diarize: note += " · Speaker labels enabled" note += " · Ctrl+C to stop" - caption_writer.print_header( - note=note if caption_format == "webvtt" else None - ) + try: + caption_writer.print_header( + note=note if caption_format == "webvtt" else None + ) + except BrokenPipeError: + live.silence_stdout() else: status.print( f"[green]✓[/green] Listening on microphone · {model} · Ctrl+C to stop" @@ -785,10 +833,9 @@ def _stream_mic( if diarize: status.print("[dim]Speaker labels enabled[/dim]") - import asyncio - try: - result = asyncio.run( + _freeze_at_exit() + return asyncio.run( self._ws_mic( client, model=model, @@ -803,42 +850,30 @@ def _stream_mic( sample_rate=sample_rate, channels=channels, caption_writer=caption_writer, + caption_format=caption_format, + save_to=save_to, + early=early, ) ) - except KeyboardInterrupt: - status.print("\n[yellow]Stopped.[/yellow]") - result = ListenResult(status="success", source="mic", mode="live") except click.ClickException: raise + except CredentialHostError: + raise except Exception as e: err = str(e) - msg = f"Microphone error: {err}" + msg_text = f"Microphone error: {err}" if ( "Invalid input device" in err or "PortAudio" in err or "No Default Input" in err ): - msg += ( + msg_text += ( "\n\nOn macOS, make sure your terminal has microphone access:" "\n System Settings → Privacy & Security → Microphone" ) - return BaseResult(status="error", message=msg) - - # Save captions or transcript - if save_to: - if caption_writer and caption_writer.accumulated_words: - content = captions_from_words( - caption_writer.accumulated_words, caption_format or "webvtt" - ) - elif result.transcript: - content = result.transcript - else: - content = "" - if content: - self._save_transcript(content, save_to) - status.print(f"[green]✓[/green] Saved to {save_to}") - - return result + return BaseResult(status="error", message=msg_text) + finally: + early.restore() async def _ws_mic( self, @@ -856,12 +891,17 @@ async def _ws_mic( sample_rate: int, channels: int, caption_writer: StreamingCaptionWriter | None = None, - ) -> ListenResult: + caption_format: str | None = None, + save_to: str | None = None, + early: live.EarlyStop | None = None, + ) -> BaseResult | ListenCodedError: import asyncio import numpy as np import sounddevice as sd + from . import live + url = self._ws_url( client, api_version=api_version, @@ -877,86 +917,86 @@ async def _ws_mic( sample_rate=sample_rate, channels=channels, ) + self._ws_host = urlsplit(url).hostname or "" api_key = client.auth_manager.get_api_key() full_transcript: list[str] = [] v2_state = self._new_v2_state() if api_version >= 2 else None - stop_event = threading.Event() - async with _live_connect(url, api_key) as ws: + loop = asyncio.get_running_loop() + queue: asyncio.Queue[Any] = asyncio.Queue() + closing = threading.Event() + stream: Any = None - async def send_audio() -> None: - loop = asyncio.get_event_loop() - q: asyncio.Queue[bytes] = asyncio.Queue() + def callback(indata: Any, frames: int, t: Any, s: Any) -> None: + if closing.is_set(): + return + if s: + print_warning( + f"Audio warning: {escape(str(s))}", + stderr=True, + ignore_quiet=True, + ) + try: + loop.call_soon_threadsafe(queue.put_nowait, bytes(indata)) + except RuntimeError: + pass # the event loop already closed + + def start_mic() -> None: + nonlocal stream + stream = sd.RawInputStream( + samplerate=sample_rate, + channels=channels, + dtype=np.int16, + callback=callback, + blocksize=int(sample_rate * 0.1), + ) + stream.start() - def callback(indata: Any, frames: int, t: Any, s: Any) -> None: - if s: - print_warning( - f"Audio warning: {escape(str(s))}", - stderr=True, - ignore_quiet=True, - ) - loop.call_soon_threadsafe(q.put_nowait, bytes(indata)) + def on_message(raw: str | bytes) -> None: + self._handle_ws_message( + raw, + full_transcript, + diarize=diarize, + interim=interim, + caption_writer=caption_writer, + v2_state=v2_state, + ) - stream = sd.RawInputStream( - samplerate=sample_rate, - channels=channels, - dtype=np.int16, - callback=callback, - blocksize=int(sample_rate * 0.1), - ) - stream.start() + try: + outcome = await live.run_session( + url, + {"Authorization": f"Token {api_key}"}, + on_message=on_message, + queue=queue, + closing=closing, + start_source=start_mic, + byte_rate=live.bytes_per_second("linear16", sample_rate, channels), + stop=early.bind(loop) if early is not None else None, + ) + if early is not None and outcome.signal is None: + outcome.signal = early.signum + finally: + closing.set() + if stream is not None: try: - while not stop_event.is_set(): - try: - data = await asyncio.wait_for(q.get(), timeout=0.5) - await ws.send(data) - except asyncio.TimeoutError: - continue - finally: stream.stop() stream.close() - await ws.send(json.dumps({"type": "CloseStream"})) - - async def recv_transcripts() -> None: - async for msg in ws: - self._handle_ws_message( - msg, - full_transcript, - diarize=diarize, - interim=interim, - caption_writer=caption_writer, - v2_state=v2_state, - ) - - send_task = asyncio.create_task(send_audio()) - recv_task = asyncio.create_task(recv_transcripts()) - try: - await asyncio.gather(send_task, recv_task) - except (KeyboardInterrupt, asyncio.CancelledError): - stop_event.set() - await _cancel_and_drain(send_task, recv_task) - except BaseException: - stop_event.set() - await _cancel_and_drain(send_task, recv_task) - raise - - # Flux may close mid-turn without an EndOfTurn; emit what we have. - if v2_state is not None: - self._flush_v2(v2_state, full_transcript, caption_writer=caption_writer) + except Exception: + pass - return ListenResult( - status="success", + return self._finish_live( + outcome, source="mic", - mode="live", model=model, language=language, - diarized=diarize, - transcript=" ".join(full_transcript), - caption_format=caption_writer.fmt if caption_writer else None, + diarize=diarize, + transcript_acc=full_transcript, + caption_writer=caption_writer, + caption_format=caption_format, + save_to=save_to, + v2_state=v2_state, ) - # ── Live stdin handler ───────────────────────────────────────────── - def _stream_stdin( self, client: DeepgramClient, @@ -975,7 +1015,11 @@ def _stream_stdin( channels: int, save_to: str | None, caption_format: str | None, - ) -> BaseResult: + ) -> BaseResult | ListenCodedError: + import asyncio + + from . import live + resolved_encoding = encoding or "linear16" if not encoding: print_warning( @@ -993,53 +1037,38 @@ def _stream_stdin( if diarize: note += " · Speaker labels enabled" note += " · Ctrl+D to finish" - caption_writer.print_header( - note=note if caption_format == "webvtt" else None - ) + try: + caption_writer.print_header( + note=note if caption_format == "webvtt" else None + ) + except BrokenPipeError: + live.silence_stdout() else: status.print( f"[green]✓[/green] Streaming stdin · {model} · send EOF (Ctrl+D) to finish" ) - import asyncio - - try: - result = asyncio.run( - self._ws_stdin( - client, - model=model, - language=language, - api_version=api_version, - diarize=diarize, - smart_format=smart_format, - punctuate=punctuate, - interim=interim, - redact=redact, - numerals=numerals, - encoding=resolved_encoding, - sample_rate=sample_rate, - channels=channels, - caption_writer=caption_writer, - ) + _freeze_at_exit() + return asyncio.run( + self._ws_stdin( + client, + model=model, + language=language, + api_version=api_version, + diarize=diarize, + smart_format=smart_format, + punctuate=punctuate, + interim=interim, + redact=redact, + numerals=numerals, + encoding=resolved_encoding, + sample_rate=sample_rate, + channels=channels, + caption_writer=caption_writer, + caption_format=caption_format, + save_to=save_to, ) - except KeyboardInterrupt: - status.print("\n[yellow]Stopped.[/yellow]") - result = ListenResult(status="success", source="stdin", mode="live") - - if save_to: - if caption_writer and caption_writer.accumulated_words: - content = captions_from_words( - caption_writer.accumulated_words, caption_format or "webvtt" - ) - elif result.transcript: - content = result.transcript - else: - content = "" - if content: - self._save_transcript(content, save_to) - status.print(f"[green]✓[/green] Saved to {save_to}") - - return result + ) async def _ws_stdin( self, @@ -1058,9 +1087,13 @@ async def _ws_stdin( sample_rate: int, channels: int, caption_writer: StreamingCaptionWriter | None = None, - ) -> ListenResult: + caption_format: str | None = None, + save_to: str | None = None, + ) -> BaseResult | ListenCodedError: import asyncio + from . import live + url = self._ws_url( client, api_version=api_version, @@ -1076,95 +1109,216 @@ async def _ws_stdin( sample_rate=sample_rate, channels=channels, ) + self._ws_host = urlsplit(url).hostname or "" api_key = client.auth_manager.get_api_key() full_transcript: list[str] = [] v2_state = self._new_v2_state() if api_version >= 2 else None - async with _live_connect(url, api_key) as ws: - loop = asyncio.get_running_loop() - audio_queue: asyncio.Queue[bytes | Exception | None] = asyncio.Queue() - stop_reader = threading.Event() - - def post_audio(item: bytes | Exception | None) -> None: - if stop_reader.is_set(): - return - try: - loop.call_soon_threadsafe(audio_queue.put_nowait, item) - except RuntimeError: - # The event loop already closed after a stream failure. - pass + loop = asyncio.get_running_loop() + queue: asyncio.Queue[Any] = asyncio.Queue() + closing = threading.Event() + reader = live.StdinReader( + loop, + queue, + closing, + read_size=live.DEFAULT_READ_BYTES, + fd=_stdin_fd(), + ) - def read_stdin() -> None: - try: - while not stop_reader.is_set(): - data = sys.stdin.buffer.read(4096) - if stop_reader.is_set(): - return - post_audio(data or None) - if not data: - return - except Exception as exc: - post_audio(exc) - - reader_thread = threading.Thread( - target=read_stdin, - name="deepctl-stdin-reader", - daemon=True, + def on_message(raw: str | bytes) -> None: + self._handle_ws_message( + raw, + full_transcript, + diarize=diarize, + interim=interim, + caption_writer=caption_writer, + v2_state=v2_state, ) - async def send_audio() -> None: - reader_thread.start() - try: - while True: - item = await audio_queue.get() - if item is None: - break - if isinstance(item, Exception): - raise item - await ws.send(item) - await ws.send(json.dumps({"type": "CloseStream"})) - finally: - stop_reader.set() - - async def recv_transcripts() -> None: - async for msg in ws: - self._handle_ws_message( - msg, - full_transcript, - diarize=diarize, - interim=interim, - caption_writer=caption_writer, - v2_state=v2_state, - ) + outcome = await live.run_session( + url, + {"Authorization": f"Token {api_key}"}, + on_message=on_message, + queue=queue, + closing=closing, + start_source=reader.start, + byte_rate=live.bytes_per_second(encoding, sample_rate, channels), + ) + return self._finish_live( + outcome, + source="stdin", + model=model, + language=language, + diarize=diarize, + transcript_acc=full_transcript, + caption_writer=caption_writer, + caption_format=caption_format, + save_to=save_to, + v2_state=v2_state, + ) - send_task = asyncio.create_task(send_audio()) - recv_task = asyncio.create_task(recv_transcripts()) - try: - await asyncio.gather(send_task, recv_task) - except (KeyboardInterrupt, asyncio.CancelledError): - # Mirror the mic path: on Ctrl-C drain the tasks but do NOT - # re-raise, so we fall through to _flush_v2 + the return below - # and the partial transcript (and --save-to) survive. Fatal - # errors still propagate via the BaseException clause. - await _cancel_and_drain(send_task, recv_task) - except BaseException: - await _cancel_and_drain(send_task, recv_task) - raise + def _finish_live( + self, + outcome: live.LiveOutcome, + *, + source: str, + model: str, + language: str, + diarize: bool, + transcript_acc: list[str], + caption_writer: StreamingCaptionWriter | None, + caption_format: str | None, + save_to: str | None, + v2_state: dict[str, Any] | None, + ) -> BaseResult | ListenCodedError: + """Turn a session outcome into the command result, after the socket + is closed: flush Flux turns, report, and run ``--save-to``.""" + from . import live # Flux may close mid-turn without an EndOfTurn; emit what we have. if v2_state is not None: - self._flush_v2(v2_state, full_transcript, caption_writer=caption_writer) + try: + self._flush_v2(v2_state, transcript_acc, caption_writer=caption_writer) + except BrokenPipeError: + live.silence_stdout() - return ListenResult( + if outcome.status == "cancelled": + status.print("\n[yellow]Stopped.[/yellow]") + if outcome.incomplete: + print_warning(msg.W_LISTEN_STOP_INCOMPLETE, stderr=True) + if outcome.signal == getattr(signal, "SIGTERM", None): + # Drained like Ctrl-C; the process then ends by SIGTERM + # (exit_code_for), so supervisors see the status they sent. + self._reraise_signal = outcome.signal + + transcript = " ".join(transcript_acc) + saved_to: str | None = None + if save_to: + if caption_writer and caption_writer.accumulated_words: + content = captions_from_words( + caption_writer.accumulated_words, caption_format or "webvtt" + ) + else: + content = transcript + if content: + self._save_transcript(content, save_to) + saved_to = save_to + status.print(f"[green]✓[/green] Saved to {save_to}") + + def kept() -> dict[str, Any]: + # The finals received so far travel with a failure. + out: dict[str, Any] = { + "transcript": transcript, + "transcript_chars": len(transcript), + } + if saved_to: + out["saved_to"] = saved_to + return out + + if outcome.status == "drain_timeout": + limit = round(outcome.drain_limit_s, 3) + template = ( + msg.E_DRAIN_CAP + if outcome.drain_reason == "cap" + else msg.E_DRAIN_TIMEOUT + ) + text = template.format(limit_s=f"{limit:g}") + return self._coded_failure( + text, + code=msg.CODE_STREAM_DRAIN_TIMEOUT, + hint_human=msg.H_DRAIN_TIMEOUT, + hint_agent=msg.H_DRAIN_TIMEOUT, + details={ + "reason": outcome.drain_reason or "idle", + "limit_s": limit, + **kept(), + }, + ) + if outcome.status == "error" and outcome.kind == "server": + server: dict[str, Any] = {} + if outcome.close_code is not None: + server["close_code"] = outcome.close_code + if outcome.request_id: + server["request_id"] = outcome.request_id + return self._coded_failure( + outcome.error or "", + code=msg.CODE_SERVER_ERROR, + hint_human=msg.H_SERVER, + hint_agent=msg.A_SERVER, + details={**server, **kept()}, + ) + if outcome.status == "error" and outcome.kind == "network": + return self._coded_failure( + outcome.error or msg.E_LISTEN_CONNECTION_LOST, + code=msg.CODE_NETWORK_UNAVAILABLE, + hint_human=msg.H_NETWORK, + hint_agent=msg.A_NETWORK.format(prog=_prog()), + details={ + "host": self._ws_host, + "reason": "connection lost", + **kept(), + }, + ) + + if outcome.status == "error": + # Reading stdin failed (EISDIR, EBADF, EIO on a hang-up). The + # fix is the same as for a missing source: pass a file or URL. + return self._coded_failure( + outcome.error or "", + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_LISTEN_STDIN_UNREADABLE, + hint_agent=msg.A_LISTEN_STDIN_UNREADABLE, + details=kept(), + next_step=f"{_prog()} {self.name} --help", + ) + + result = ListenResult( status="success", - source="stdin", + source=source, mode="live", model=model, language=language, diarized=diarize, - transcript=" ".join(full_transcript), + transcript=transcript, caption_format=caption_writer.fmt if caption_writer else None, ) + if outcome.status == "cancelled" and not ( + source == "mic" and outcome.signal == signal.SIGINT + ): + # Ctrl-C is how a mic session normally ends: the transcript so + # far is the result, exit 0. Other sources, and SIGTERM for + # every source, stay "cancelled". Ctrl-Break on Windows is a + # hard stop like SIGTERM, so it stays "cancelled" on mic too. + result.status = "cancelled" + return result + + def _coded_failure( + self, + text: str, + *, + code: str, + hint_human: str, + hint_agent: str, + details: dict[str, Any], + next_step: str | None = None, + ) -> ListenCodedError: + """Report a live-stream failure on stderr and return its envelope.""" + hint = hint_agent if (_agentic or is_ai_agent()) else hint_human + print_error(escape(text)) + print_hint(escape(hint)) + return ListenCodedError( + error=text, code=code, hint=hint, next_step=next_step, details=details + ) + + def exit_code_for(self, result: Any) -> int: + """A SIGTERM-drained stream ends by SIGTERM itself: the + result is already written, so flush and re-raise it here.""" + if self._reraise_signal is not None: + signum, self._reraise_signal = self._reraise_signal, None + from . import live + + live.die_by_signal(signum) + return super().exit_code_for(result) # ── Shared WebSocket helpers ─────────────────────────────────────── @@ -1256,7 +1410,9 @@ def _handle_ws_message( if msg_type == "Error" and v2_state is not None: code = data.get("code") or "UNKNOWN_ERROR" description = data.get("description") or "No description provided" - raise click.ClickException(f"Flux STT error ({code}): {description}") + raise click.ClickException( + msg.E_LISTEN_FLUX_ERROR.format(flux_code=code, description=description) + ) if msg_type != "Results": return diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py new file mode 100644 index 00000000..9e7378f2 --- /dev/null +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py @@ -0,0 +1,673 @@ +"""Live streaming engine for ``dg listen`` (stdin and microphone). + +One WebSocket session: audio goes up from a queue, transcripts come +down to a callback, and the session ends one of four ways: + +- the server closes the stream (exit 0, or 1 if it reported an error); +- stdin reaches end of input: ``CloseStream``, then wait for the server to + close under an idle deadline and an absolute cap (``stream_drain_timeout`` + if either expires); +- Ctrl-C (or Ctrl-Break on Windows): ``CloseStream``, then wait at most + ``STOP_DRAIN_S`` for the server's last results (exit 2). On ``--mic``, + Ctrl-C is how the session ends, so it exits 0; Ctrl-Break stays a hard + stop (exit 2). SIGTERM on POSIX drains the same way, and the command + then re-raises it. A second signal ends the process at once; +- the connection fails (exit 1). + +Stdin is read with ``os.read(0, cap)`` in a daemon thread that hands each +buffer to the event loop and waits until the queue accepts it. The thread is +never joined. Before every read and every hand-off it checks ``closing``, +which shutdown sets before it cancels any task; anything raised in the thread +(including a cancelled hand-off while the loop shuts down) ends the thread +quietly. ``loop.connect_read_pipe`` isn't used: it rejects a regular file on +stdin, misbehaves on ``/dev/null``, sets ``O_NONBLOCK`` on a descriptor shared +with the parent, and doesn't work with the Windows Proactor loop. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import re +import signal +import threading +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import click + +from . import messages as msg + +if TYPE_CHECKING: + from collections.abc import Callable + +# ── Tunables ──────────────────────────────────────────────────────────────── + +#: Largest single read from stdin. +DEFAULT_READ_BYTES = 4096 +#: Ctrl-C: how long to wait for the server's last results after CloseStream. +STOP_DRAIN_S = 1.0 +#: End of input: give up when the server has sent nothing for this long. +EOF_IDLE_S = 10.0 +#: End of input: the absolute cap is max(EOF_CAP_MIN_S, 2 x the work left). +EOF_CAP_MIN_S = 30.0 +#: Bound on the WebSocket closing handshake once the outcome is known. +CLOSE_TIMEOUT_S = 1.0 + +#: Bytes per sample for encodings whose byte rate is known from the rate. +_PCM_WIDTH = {"linear16": 2, "linear32": 4, "mulaw": 1, "alaw": 1} + +CLOSE_STREAM = json.dumps({"type": "CloseStream"}) + + +def bytes_per_second(encoding: str | None, sample_rate: int, channels: int) -> int: + """Byte rate of raw PCM, mulaw or alaw audio; 0 when it isn't knowable.""" + width = _PCM_WIDTH.get((encoding or "").lower()) + if not width or sample_rate <= 0 or channels <= 0: + return 0 + return width * sample_rate * channels + + +# ── Audio sources ─────────────────────────────────────────────────────────── + + +class StdinReader: + """Reads a file descriptor in a daemon thread into a queue. + + Puts ``bytes`` for audio, ``b""`` once at end of input, or the + ``OSError`` a read raised. Never joined: a read blocked on an idle pipe + can't be interrupted portably, and the process may exit while it waits. + """ + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + queue: asyncio.Queue[Any], + closing: threading.Event, + *, + read_size: int, + fd: int = 0, + ) -> None: + self._loop = loop + self._queue = queue + self._closing = closing + self._read_size = read_size + self._fd = fd + + def start(self) -> None: + threading.Thread( + target=self._run, name="deepctl-stdin-reader", daemon=True + ).start() + + def _put(self, item: Any) -> bool: + """Hand ``item`` to the loop and wait for the queue to take it.""" + if self._closing.is_set() or self._loop.is_closed(): + return False + coro = self._queue.put(item) + try: + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + except BaseException: + # The loop closed between the check and the call. + coro.close() + return False + future.result() + return True + + def _run(self) -> None: + try: + # Keep Ctrl-C on the main thread, where the handler runs; a + # signal routed here would only interrupt the read. + if hasattr(signal, "pthread_sigmask"): + signal.pthread_sigmask( + signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM} + ) + while not self._closing.is_set(): + try: + data = os.read(self._fd, self._read_size) + except BlockingIOError: + # The parent left O_NONBLOCK on the shared descriptor. + # Wait for data rather than change its flags. + _wait_readable(self._fd) + continue + except OSError as exc: + self._put(exc) + return + if self._closing.is_set(): + return + if not self._put(data) or not data: + return + except BaseException: + # Shutdown cancelled the hand-off, or the loop is gone. Either + # way the session no longer wants audio. + return + + +def _wait_readable(fd: int) -> None: + """Block (in short slices) until ``fd`` has data or is at end of input.""" + import select + + try: + select.select([fd], [], [], 0.2) + except (OSError, ValueError): + time.sleep(0.2) + + +# ── Session ───────────────────────────────────────────────────────────────── + + +@dataclass +class LiveOutcome: + """How a live session ended.""" + + #: "success" (server closed), "cancelled" (Ctrl-C or SIGTERM), "error", + #: or "drain_timeout" (end of input, the server never finished). + status: str = "success" + #: Error text for "error" (already scrubbed and safe to print). + error: str | None = None + #: For "error": "server" (error close or error frame), "network" (the + #: connection dropped) or "read" (stdin failed). + kind: str | None = None + #: The server's close code, for a "server" error that closed the socket. + close_code: int | None = None + #: The request id the server reported, when it sent one. + request_id: str | None = None + #: The signal that stopped a "cancelled" session. + signal: int | None = None + #: The reader of stdout went away; the session ended quietly. + stdout_closed: bool = False + #: Ctrl-C: True when the server hadn't finished within STOP_DRAIN_S. + incomplete: bool = False + #: "idle" or "cap" for a drain timeout. + drain_reason: str | None = None + #: The bound that expired, in seconds, for a drain timeout. + drain_limit_s: float = 0.0 + + +@dataclass +class _Clock: + last_server: float = 0.0 + last_audio: float = 0.0 + first_send: float | None = None + eof_at: float | None = None + bytes_sent: int = 0 + last_final_end: float = 0.0 + + +def _final_end(data: dict[str, Any]) -> float | None: + """Audio time covered by a final result, if this message is one.""" + if data.get("type") == "Results" and data.get("is_final"): + try: + return float(data.get("start", 0.0)) + float(data.get("duration", 0.0)) + except (TypeError, ValueError): + return None + if data.get("type") == "TurnInfo" and data.get("event") == "EndOfTurn": + try: + return float(data.get("audio_window_end", 0.0)) + except (TypeError, ValueError): + return None + return None + + +_AUTH_RE = re.compile(r"(?i)\b(token|bearer)\s+\S+") + + +def _clean(text: object, limit: int = 200, secrets: tuple[str, ...] = ()) -> str: + """Server-supplied text for a message: one line, printable, no + credentials, bounded. Secrets are scrubbed before the cut, so a + truncation can't leave part of one.""" + out = "".join(ch if ch.isprintable() else " " for ch in str(text)) + for secret in secrets: + if len(secret) >= 8: # a real credential, not a stray letter + out = out.replace(secret, "****" + secret[-4:]) + out = _AUTH_RE.sub(r"\1 ****", out) + return out.strip()[:limit] + + +def _closed_outcome(exc: BaseException, secrets: tuple[str, ...]) -> LiveOutcome | None: + """The outcome for a connection that ended, or None for a clean close.""" + try: + from websockets.exceptions import ConnectionClosed, ConnectionClosedOK + except Exception: # pragma: no cover - websockets is a hard dependency + return LiveOutcome( + status="error", kind="network", error=_clean(exc, 200, secrets) + ) + if isinstance(exc, ConnectionClosedOK): + return None + if isinstance(exc, ConnectionClosed): + rcvd = getattr(exc, "rcvd", None) + if rcvd is not None: + reason = _clean(getattr(rcvd, "reason", ""), 200, secrets) + code = getattr(rcvd, "code", None) + text = ( + msg.E_LISTEN_SERVER_CLOSED.format(close_code=code, close_reason=reason) + if reason + else msg.E_LISTEN_SERVER_CLOSED_NO_REASON.format(close_code=code) + ) + return LiveOutcome( + status="error", + kind="server", + error=text, + close_code=code if isinstance(code, int) else None, + ) + return LiveOutcome( + status="error", kind="network", error=msg.E_LISTEN_CONNECTION_LOST + ) + return LiveOutcome( + status="error", + kind="network", + error=_clean(exc, 200, secrets) or exc.__class__.__name__, + ) + + +def silence_stdout() -> None: + """The reader of stdout went away (EPIPE): point fd 1 at the null + device, so buffered output and later writes, including the flush at + exit, succeed quietly.""" + try: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 1) + os.close(devnull) + except OSError: + pass + + +def _is_connection_closed(exc: BaseException) -> bool: + try: + from websockets.exceptions import ConnectionClosed + except Exception: # pragma: no cover - websockets is a hard dependency + return False + return isinstance(exc, ConnectionClosed) + + +class _SignalGuard: + """Ctrl-C (and Ctrl-Break on Windows) and, on POSIX, SIGTERM → ``stop``. + + The first signal starts the drain; ``signum`` records which one it was. + A second signal ends the process at once: by SIGTERM if either signal + was SIGTERM, else with exit 2. Uses ``signal.signal`` rather than + ``loop.add_signal_handler``, which is Unix-only. Outside the main thread + (tests) it installs nothing. + """ + + def __init__(self, loop: asyncio.AbstractEventLoop, stop: asyncio.Event) -> None: + self._loop = loop + self._stop = stop + self._count = 0 + self._previous: dict[int, Any] = {} + self.signum: int | None = None + + def _handler(self, signum: int, _frame: Any) -> None: + self._count += 1 + if self._count == 1: + self.signum = signum + try: + self._loop.call_soon_threadsafe(self._stop.set) + return + except BaseException: + pass + # Second signal, or the loop is already gone: leave without + # draining. stdout was flushed line by line. + sigterm = getattr(signal, "SIGTERM", None) + if sigterm is not None and sigterm in (signum, self.signum) and os.name != "nt": + die_by_signal(sigterm) + os._exit(2) + + def install(self) -> None: + if threading.current_thread() is not threading.main_thread(): + return + sigs = [signal.SIGINT] + sigbreak = getattr(signal, "SIGBREAK", None) + if sigbreak is not None: + sigs.append(sigbreak) + if os.name != "nt": + sigs.append(signal.SIGTERM) + for sig in sigs: + try: + self._previous[sig] = signal.signal(sig, self._handler) + except (OSError, ValueError): + pass + + def restore(self) -> None: + for sig, previous in self._previous.items(): + try: + signal.signal(sig, previous) + except (OSError, ValueError, TypeError): + pass + self._previous.clear() + + +class EarlyStop: + """Ctrl-C for a mic session before ``run_session`` installs its guard. + + Installed before the banner, so a Ctrl-C during the imports or while + connecting is recorded instead of raising ``KeyboardInterrupt``; once + ``bind`` hands over the session's ``stop`` event, it ends the session + like the guard's first signal would. A second Ctrl-C exits at once. + asyncio.run leaves a non-default SIGINT handler alone, so this one stays + in place until ``restore``. Outside the main thread it does nothing. + """ + + def __init__(self) -> None: + self.signum: int | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._stop: asyncio.Event | None = None + self._previous: Any = None + self._installed = False + + def _handler(self, signum: int, _frame: Any) -> None: + if self.signum is not None: + os._exit(2) + self.signum = signum + if self._loop is not None and self._stop is not None: + try: + self._loop.call_soon_threadsafe(self._stop.set) + except RuntimeError: + pass + + def install(self) -> None: + if threading.current_thread() is not threading.main_thread(): + return + try: + self._previous = signal.signal(signal.SIGINT, self._handler) + self._installed = True + except (OSError, ValueError): + pass + + def bind(self, loop: asyncio.AbstractEventLoop) -> asyncio.Event: + """The ``stop`` event for ``run_session``; already set if Ctrl-C came.""" + self._stop = asyncio.Event() + self._loop = loop + if self.signum is not None: + self._stop.set() + return self._stop + + def restore(self) -> None: + if self._installed: + try: + signal.signal(signal.SIGINT, self._previous) + except (OSError, ValueError, TypeError): + pass + self._installed = False + + +def die_by_signal(signum: int) -> None: + """End the process by ``signum`` with its default action, so the wait + status is "killed by signal N" (shells report 128+N). + + PID 1 in a container ignores a default-action signal, so if the process + is still here afterwards it exits with the shell's 128+N. + """ + import sys + + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except Exception: + pass + try: + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + time.sleep(0.05) + except Exception: + pass + os._exit(128 + signum) + + +async def _cancel_all(*tasks: asyncio.Task[Any] | None) -> None: + live = [t for t in tasks if t is not None] + for task in live: + if not task.done(): + task.cancel() + for task in live: + try: + await task + except BaseException: + pass + + +def _open(url: str, headers: dict[str, str], **options: Any) -> Any: + """Open the live socket; a redirect on the upgrade, to the same origin + or another, raises ``RedirectRefused`` before a second connection, so + the key and the audio go only to the host that passed the host check.""" + from deepctl_core.websocket import open_async + + return open_async(url, headers, command="dg listen", payload="audio", **options) + + +async def run_session( + url: str, + headers: dict[str, str], + *, + on_message: Callable[[str | bytes], None], + queue: asyncio.Queue[Any], + closing: threading.Event, + start_source: Callable[[], None], + byte_rate: int, + stop: asyncio.Event | None = None, +) -> LiveOutcome: + """Run one live session; see the module docstring for how it ends. + + ``queue`` carries audio ``bytes``, ``b""`` at end of input, or an + exception from the source. ``start_source`` starts the producer once the + socket is open. ``byte_rate`` is the audio byte rate when it is known + (raw PCM, mulaw, alaw), else 0. ``closing`` is set before any task is + cancelled, so the producer stops handing over audio first. + """ + loop = asyncio.get_running_loop() + stop = stop or asyncio.Event() + guard = _SignalGuard(loop, stop) + guard.install() + clock = _Clock() + now = time.monotonic + # Values that must never be echoed back (the credential in the header). + secrets = tuple(v.split(" ", 1)[-1] for v in headers.values()) + request_id: str | None = None + ws: Any = None + tasks: dict[str, asyncio.Task[Any] | None] = {} + last: list[LiveOutcome] = [] + + def track(raw: str | bytes) -> None: + nonlocal request_id + clock.last_server = now() + try: + data = json.loads(raw) + except Exception: + return + if isinstance(data, dict): + rid = data.get("request_id") or (data.get("metadata") or {}).get( + "request_id" + ) + if isinstance(rid, str) and not request_id: + request_id = _clean(rid, 100, secrets) + end = _final_end(data) + if end is not None: + clock.last_final_end = max(clock.last_final_end, end) + + async def receive() -> LiveOutcome | None: + """Read until the server closes. Returns the failure, or None.""" + try: + async for raw in ws: + track(raw) + on_message(raw) + except BrokenPipeError: + # Whoever read stdout is gone (``| head -n 1``): end quietly. + silence_stdout() + return LiveOutcome(stdout_closed=True) + except click.ClickException as exc: + # A server-reported error frame (Flux "Error"). + return LiveOutcome( + status="error", kind="server", error=_clean(exc.message, 200, secrets) + ) + except Exception as exc: + return _closed_outcome(exc, secrets) + return None + + def ended(result: LiveOutcome | None) -> LiveOutcome: + out = result or LiveOutcome() + out.request_id = request_id + last.append(out) + return out + + async def send_audio() -> str: + """Send audio until end of input, then CloseStream. Returns "eof".""" + while True: + item = await queue.get() + if isinstance(item, BaseException): + raise item + if not item: + break + await ws.send(item) + t = now() + clock.last_audio = t + if clock.first_send is None: + clock.first_send = t + clock.bytes_sent += len(item) + clock.eof_at = now() + await ws.send(CLOSE_STREAM) + return "eof" + + def eof_cap() -> float: + if byte_rate: + behind = clock.bytes_sent / byte_rate - clock.last_final_end + else: + start = clock.first_send if clock.first_send is not None else clock.eof_at + behind = (clock.eof_at or now()) - (start or now()) + return max(EOF_CAP_MIN_S, 2.0 * max(0.0, behind)) + + async def finish_on_stop(deadline: float) -> LiveOutcome: + """Ctrl-C: stop sending, CloseStream, wait for the server until deadline.""" + closing.set() + await _cancel_all(tasks.get("send")) + recv = tasks["recv"] + assert recv is not None + if clock.eof_at is None and not recv.done(): + try: + await asyncio.wait_for( + ws.send(CLOSE_STREAM), max(0.0, deadline - now()) + ) + except BaseException: + pass + if not recv.done(): + await asyncio.wait({recv}, timeout=max(0.0, deadline - now())) + done = recv.done() and not recv.cancelled() + return LiveOutcome( + status="cancelled", + incomplete=not done, + signal=guard.signum, + request_id=request_id, + ) + + try: + stop_wait = asyncio.ensure_future(stop.wait()) + tasks["stop"] = stop_wait + connect = asyncio.ensure_future( + _open(url, headers, close_timeout=CLOSE_TIMEOUT_S) + ) + tasks["connect"] = connect + await asyncio.wait({connect, stop_wait}, return_when=asyncio.FIRST_COMPLETED) + if not connect.done(): + closing.set() + await _cancel_all(connect) + return LiveOutcome(status="cancelled", signal=guard.signum) + ws = connect.result() + clock.last_server = clock.last_audio = now() + + start_source() + recv = asyncio.ensure_future(receive()) + send = asyncio.ensure_future(send_audio()) + tasks.update(recv=recv, send=send) + + # ── Streaming ───────────────────────────────────────────────── + await asyncio.wait({recv, send, stop_wait}, return_when=asyncio.FIRST_COMPLETED) + + if stop_wait.done(): + return await finish_on_stop(now() + STOP_DRAIN_S) + + if recv.done(): + return ended(recv.result()) + + # The sender finished first: end of input, or a failure. + try: + send.result() + except Exception as exc: + if _is_connection_closed(exc): + # The socket went away while sending; the receiver has the + # reason (or soon will). + await asyncio.wait({recv}, timeout=CLOSE_TIMEOUT_S) + return ended( + recv.result() if recv.done() else _closed_outcome(exc, secrets) + ) + reason = getattr(exc, "strerror", None) or exc + return ended( + LiveOutcome( + status="error", + kind="read", + error=msg.E_LISTEN_STDIN_UNREADABLE.format( + os_error=_clean(reason, 200, secrets) + ), + ) + ) + + # ── End of input: drain ─────────────────────────────────────── + assert clock.eof_at is not None + cap = eof_cap() + cap_at = clock.eof_at + cap + while True: + idle_at = max(clock.eof_at, clock.last_server) + EOF_IDLE_S + wake = min(idle_at, cap_at) + remaining = wake - now() + if remaining <= 0: + closing.set() + if cap_at <= idle_at: + return ended( + LiveOutcome( + status="drain_timeout", + drain_reason="cap", + drain_limit_s=cap, + ) + ) + return ended( + LiveOutcome( + status="drain_timeout", + drain_reason="idle", + drain_limit_s=EOF_IDLE_S, + ) + ) + await asyncio.wait( + {recv, stop_wait}, + timeout=remaining, + return_when=asyncio.FIRST_COMPLETED, + ) + if recv.done(): + return ended(recv.result()) + if stop_wait.done(): + return await finish_on_stop(now() + STOP_DRAIN_S) + finally: + # Order matters: stop the producer before cancelling anything, so a + # hand-off cancelled below can't surface in the reader thread. + closing.set() + await _cancel_all(*(t for k, t in tasks.items() if k != "recv")) + recv_task = tasks.get("recv") + if ws is not None: + if recv_task is None or not recv_task.done(): + # The server hasn't closed: don't wait for its handshake. + transport = getattr(ws, "transport", None) + if transport is not None: + transport.abort() + close = getattr(ws, "close", None) + if close is not None: + try: + await asyncio.wait_for(close(), CLOSE_TIMEOUT_S) + except BaseException: + pass + await _cancel_all(recv_task) + guard.restore() + # A signal that landed while the server was closing still counts: + # the clean close becomes a stop by that signal. + if guard.signum is not None: + for out in last: + if out.status == "success" and not out.stdout_closed: + out.status = "cancelled" + out.signal = guard.signum diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py new file mode 100644 index 00000000..6011ac93 --- /dev/null +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py @@ -0,0 +1,72 @@ +"""User-facing strings for ``dg listen``'s live path, keyed by message ID. + +Every ID and string matches the error catalogue (``error_codes.json``) +exactly; the attribute name is the ID with ``-`` as ``_``. +""" + +from __future__ import annotations + +# ── Catalogue ────────────────────────────────────────────────────────────── + +CODE_INVALID_ARGUMENT = "invalid_argument" +CODE_STREAM_DRAIN_TIMEOUT = "stream_drain_timeout" +CODE_SERVER_ERROR = "server_error" +CODE_NETWORK_UNAVAILABLE = "network_unavailable" + +# invalid_argument: reading stdin failed. +E_LISTEN_STDIN_UNREADABLE = "Couldn't read audio from stdin: {os_error}." +H_LISTEN_STDIN_UNREADABLE = ( + "Pass a file path or URL instead, or make sure stdin is an open pipe " + "or file with audio in it." +) +A_LISTEN_STDIN_UNREADABLE = ( + "Pass the audio file path or URL as SOURCE instead of reading stdin, then retry." +) + +# stream_drain_timeout: end of input, then the idle deadline or the cap. +E_DRAIN_TIMEOUT = ( + "The server sent nothing for {limit_s} s after the audio ended, " + "so results may be incomplete." +) +E_DRAIN_CAP = ( + "The server was still sending results {limit_s} s after the audio ended, " + "so results may be incomplete." +) +H_DRAIN_TIMEOUT = "Results received so far were kept. Try again if you need the rest." + +# server_error / network_unavailable hints (the error text is the stream's own). +H_SERVER = ( + "Try again in a moment. If it keeps failing, contact Deepgram support " + "with the request ID in details." +) +A_SERVER = ( + "Retry once after a few seconds. If it fails again, tell the user and " + "include details.request_id." +) +H_NETWORK = "Check your network connection and proxy settings, then try again." +A_NETWORK = ( + "{prog} can't reach Deepgram from this environment. If you run in a " + "sandbox, ask the user to allow network access for {prog}." +) + +# ── Live-stream messages ─────────────────────────────────────────────────── + +# Ctrl-C or SIGTERM: the server didn't finish within the 1 s drain. +W_LISTEN_STOP_INCOMPLETE = ( + "The server hadn't sent its last results after 1 s, " + "so the transcript may be incomplete." +) + +# server_error: the server closed the stream with an error close code. +E_LISTEN_SERVER_CLOSED = ( + "The server ended the stream with an error ({close_code}): {close_reason}" +) +E_LISTEN_SERVER_CLOSED_NO_REASON = ( + "The server ended the stream with an error ({close_code})." +) + +# server_error: Flux (listen v2) reported an error in an Error frame. +E_LISTEN_FLUX_ERROR = "Flux STT error ({flux_code}): {description}" + +# network_unavailable: the connection dropped without a closing handshake. +E_LISTEN_CONNECTION_LOST = "The connection to the server was lost." diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/models.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/models.py index f05f5921..f79214b7 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/models.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/models.py @@ -4,7 +4,7 @@ from typing import Any -from deepctl_core import BaseResult +from deepctl_core import BaseResult, CodedErrorResult class ListenResult(BaseResult): @@ -30,3 +30,20 @@ class ListenResult(BaseResult): full_result: dict[str, Any] | None = None # raw Deepgram API response probe_info: dict[str, Any] | None = None duration_seconds: float = 0.0 + + +class ListenCodedError(CodedErrorResult): + """A coded error from ``dg listen``, keys in the catalogue's order. + + ``details`` is left out of the payload when there are none. + """ + + next_step: str | None = None + schema_version: int = 1 + details: dict[str, Any] | None = None + + def model_dump(self, **kwargs: Any) -> dict[str, Any]: + data = super().model_dump(**kwargs) + if data.get("details") is None: + data.pop("details", None) + return data diff --git a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py index c0fecbfb..f3e2da3d 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py @@ -17,6 +17,7 @@ def _install_flux_error_websockets(monkeypatch, ready=None): + """A server that sends one Flux Error frame (after ``ready()``), then ends.""" import asyncio import json import sys @@ -31,12 +32,17 @@ def _install_flux_error_websockets(monkeypatch, ready=None): ) class _FakeWS: + transport = None + def __init__(self): self.messages = iter([error_frame]) async def send(self, _data): return None + async def close(self): + return None + def __aiter__(self): return self @@ -48,17 +54,70 @@ async def __anext__(self): except StopIteration: raise StopAsyncIteration from None - class _FakeConnect: - async def __aenter__(self): - return _FakeWS() + _install_websockets(monkeypatch, _FakeWS()) - async def __aexit__(self, *_args): - return False - monkeypatch.setattr( - "deepctl_cmd_listen.command._live_connect", - lambda *_args, **_kwargs: _FakeConnect(), - ) +def _install_websockets(monkeypatch, ws): + """Make ``await websockets.connect(...)`` return ``ws``.""" + import sys + import types + + class _Connect: + def __await__(self): + async def _open(): + return ws + + return _open().__await__() + + fake = types.ModuleType("websockets") + fake.connect = lambda *_args, **_kwargs: _Connect() + monkeypatch.setitem(sys.modules, "websockets", fake) + monkeypatch.setattr("deepctl_cmd_listen.live._open", fake.connect) + + +def _stdin_pipe(monkeypatch, data=b"", *, close=True): + """Point the live stdin reader at a pipe holding ``data``. + + Returns the write end when ``close`` is False (the pipe stays open, so + the reader blocks), else None. + """ + import os + + r, w = os.pipe() + if data: + os.write(w, data) + if close: + os.close(w) + w = None + monkeypatch.setattr("deepctl_cmd_listen.command._stdin_fd", lambda: r) + return w + + +class _EndOnCloseStream: + """A server that ends the stream when CloseStream arrives.""" + + transport = None + + def __init__(self): + import asyncio + + self.sent = [] + self.ended = asyncio.Event() + + async def send(self, data): + self.sent.append(data) + if "CloseStream" in str(data): + self.ended.set() + + async def close(self): + self.ended.set() + + def __aiter__(self): + return self + + async def __anext__(self): + await self.ended.wait() + raise StopAsyncIteration class TestListenCommand: @@ -365,6 +424,7 @@ def test_click_flux_error_frame_exits_nonzero( ): """A stdin Error frame propagates through the stream and Click boundary.""" _install_flux_error_websockets(monkeypatch) + _stdin_pipe(monkeypatch) result = self.invoke_click( command, @@ -961,46 +1021,39 @@ def test_captions_use_audio_window_and_survive_batch_save(self, command): assert "my account" in batch assert "00:00:01,000 --> 00:00:02,500" in batch - def test_ws_mic_flushes_final_turn_on_keyboard_interrupt( - self, command, monkeypatch - ): - """Ctrl-C during a Flux mic stream must still flush the in-flight turn. - - The interrupt surfaces at the ``asyncio.gather`` await; ``_ws_mic`` has - to catch it, run ``_flush_v2``, and return the accumulated transcript - (rather than letting the final turn vanish).""" + def test_ws_mic_flushes_final_turn_on_ctrl_c(self, command, monkeypatch): + """Ctrl-C during a Flux mic stream must still flush the in-flight turn, + with CloseStream sent. Ctrl-C is how a mic session ends, so the + result is a success (exit 0), as in 0.4.0.""" import asyncio as _asyncio + import signal import sys import types from unittest.mock import MagicMock - # _ws_mic imports these at call time; stub them so no hardware/net is hit. - for name in ("sounddevice", "numpy"): - monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) - - class _FakeWS: - async def send(self, *a): - return None - - def __aiter__(self): - return self + class _FakeInputStream: + def __init__(self, **_kwargs): + pass - async def __anext__(self): - raise StopAsyncIteration + def start(self): + pass - class _FakeConnect: - async def __aenter__(self): - return _FakeWS() + def stop(self): + pass - async def __aexit__(self, *a): - return False + def close(self): + pass - monkeypatch.setattr( - "deepctl_cmd_listen.command._live_connect", - lambda *a, **k: _FakeConnect(), - ) + fake_sd = types.ModuleType("sounddevice") + fake_sd.RawInputStream = _FakeInputStream + fake_np = types.ModuleType("numpy") + fake_np.int16 = "int16" + monkeypatch.setitem(sys.modules, "sounddevice", fake_sd) + monkeypatch.setitem(sys.modules, "numpy", fake_np) + ws = _EndOnCloseStream() + _install_websockets(monkeypatch, ws) - # A turn received but never finalized (no EndOfTurn before the interrupt). + # A turn received but never finalized (no EndOfTurn before Ctrl-C). seeded = { "turns": { 0: { @@ -1015,18 +1068,12 @@ async def __aexit__(self, *a): } monkeypatch.setattr(command, "_new_v2_state", lambda: seeded) - async def _interrupt(*a, **k): - raise KeyboardInterrupt - - monkeypatch.setattr("deepctl_cmd_listen.command.asyncio.gather", _interrupt) - client = MagicMock() client.config.get_profile.return_value.base_url = "https://api.deepgram.com" client.auth_manager.get_api_key.return_value = "k" - # Drive on a plain loop (not asyncio.run) so the patched gather can't - # interfere with run()'s own shutdown machinery. loop = _asyncio.new_event_loop() + loop.call_later(0.1, signal.raise_signal, signal.SIGINT) try: result = loop.run_until_complete( command._ws_mic( @@ -1048,6 +1095,9 @@ async def _interrupt(*a, **k): loop.close() assert result.transcript == "hello world" + assert result.status == "success" + assert command.exit_code_for(result) == 0 + assert '{"type": "CloseStream"}' in ws.sent def test_end_of_turn_with_empty_transcript_is_noop(self, command, capsys): """An EndOfTurn that never carried text emits nothing (no blank line).""" @@ -1173,97 +1223,48 @@ def close(self): client.config.get_profile.return_value.base_url = "https://api.deepgram.com" client.auth_manager.get_api_key.return_value = "test-key" - with pytest.raises(click.ClickException) as exc_info: - command._stream_mic( - client, - model="flux-general-en", - language="en-US", - api_version=2, - diarize=False, - smart_format=True, - punctuate=True, - interim=False, - redact=(), - numerals=False, - sample_rate=16000, - channels=1, - save_to=None, - caption_format=None, - ) + result = command._stream_mic( + client, + model="flux-general-en", + language="en-US", + api_version=2, + diarize=False, + smart_format=True, + punctuate=True, + interim=False, + redact=(), + numerals=False, + sample_rate=16000, + channels=1, + save_to=None, + caption_format=None, + ) - assert "INVALID_AUDIO" in exc_info.value.message - assert "audio stream is invalid" in exc_info.value.message + assert result.status == "error" + assert result.code == "server_error" + assert command.exit_code_for(result) == 1 + assert "INVALID_AUDIO" in result.error + assert "audio stream is invalid" in result.error assert events == {"started": True, "stopped": True, "closed": True} def test_stream_stdin_flux_error_does_not_wait_for_blocked_input( self, command, monkeypatch ): - import asyncio as _asyncio - import json as _json - import sys + """A server error ends the stream while stdin is still open and idle.""" + import os import threading - import types from unittest.mock import MagicMock - read_started = threading.Event() - release_read = threading.Event() - - class _BlockingBuffer: - def read(self, _size): - read_started.set() - release_read.wait() - return b"" - - fake_stdin = types.SimpleNamespace(buffer=_BlockingBuffer()) - monkeypatch.setattr( - "deepctl_cmd_listen.command.sys.stdin", fake_stdin, raising=False - ) - - error_frame = _json.dumps( - { - "type": "Error", - "code": "INVALID_AUDIO", - "description": "audio stream is invalid", - } - ) - - class _FakeWS: - def __init__(self): - self.sent_error = False - - async def send(self, _data): - return None - - def __aiter__(self): - return self - - async def __anext__(self): - if self.sent_error: - raise StopAsyncIteration - while not read_started.is_set(): - await _asyncio.sleep(0) - self.sent_error = True - return error_frame - - class _FakeConnect: - async def __aenter__(self): - return _FakeWS() - - async def __aexit__(self, *_args): - return False - - monkeypatch.setattr( - "deepctl_cmd_listen.command._live_connect", - lambda *_args, **_kwargs: _FakeConnect(), - ) + writer = _stdin_pipe(monkeypatch, close=False) + _install_flux_error_websockets(monkeypatch) client = MagicMock() client.config.get_profile.return_value.base_url = "https://api.deepgram.com" client.auth_manager.get_api_key.return_value = "test-key" - errors = [] + results = [] def run_stream(): - try: + results.append( command._stream_stdin( client, model="flux-general-en", @@ -1281,61 +1282,34 @@ def run_stream(): save_to=None, caption_format=None, ) - except BaseException as exc: - errors.append(exc) + ) stream_thread = threading.Thread(target=run_stream, daemon=True) stream_thread.start() - assert read_started.wait(timeout=1) - stream_thread.join(timeout=1) - exited_before_stdin = not stream_thread.is_alive() - release_read.set() - stream_thread.join(timeout=1) - - assert exited_before_stdin - assert len(errors) == 1 - assert isinstance(errors[0], click.ClickException) - assert "INVALID_AUDIO" in errors[0].message - - def test_stream_stdin_keyboard_interrupt_preserves_transcript_and_save_to( + stream_thread.join(timeout=2) + exited_while_stdin_open = not stream_thread.is_alive() + os.close(writer) + + assert exited_while_stdin_open + (result,) = results + assert result.status == "error" + assert result.code == "server_error" + assert "INVALID_AUDIO" in result.error + + def test_stream_stdin_ctrl_c_preserves_transcript_and_save_to( self, command, monkeypatch, tmp_path ): """Ctrl-C on `dg listen -` (stdin) keeps the partial transcript and - writes --save-to, mirroring the mic path. The stdin path used to - re-raise the interrupt and drop both.""" - import sys - import types + writes --save-to, and ends as cancelled (exit 2).""" + import os + import signal + import threading from unittest.mock import MagicMock - # No real stdin read: return EOF at once if the reader thread starts. - fake_stdin = types.SimpleNamespace( - buffer=types.SimpleNamespace(read=lambda _n: b"") - ) - monkeypatch.setattr( - "deepctl_cmd_listen.command.sys.stdin", fake_stdin, raising=False - ) - - class _FakeWS: - async def send(self, *a): - return None - - def __aiter__(self): - return self - - async def __anext__(self): - raise StopAsyncIteration - - class _FakeConnect: - async def __aenter__(self): - return _FakeWS() - - async def __aexit__(self, *a): - return False - - monkeypatch.setattr( - "deepctl_cmd_listen.command._live_connect", - lambda *a, **k: _FakeConnect(), - ) + writer = _stdin_pipe(monkeypatch, close=False) + ws = _EndOnCloseStream() + _install_websockets(monkeypatch, ws) + threading.Timer(0.2, os.kill, (os.getpid(), signal.SIGINT)).start() # A turn received but never finalized before the interrupt. seeded = { @@ -1352,11 +1326,6 @@ async def __aexit__(self, *a): } monkeypatch.setattr(command, "_new_v2_state", lambda: seeded) - async def _interrupt(*a, **k): - raise KeyboardInterrupt - - monkeypatch.setattr("deepctl_cmd_listen.command.asyncio.gather", _interrupt) - client = MagicMock() client.config.get_profile.return_value.base_url = "https://api.deepgram.com" client.auth_manager.get_api_key.return_value = "k" @@ -1380,5 +1349,8 @@ async def _interrupt(*a, **k): caption_format=None, ) + os.close(writer) assert result.transcript == "hello world" + assert result.status == "cancelled" + assert command.exit_code_for(result) == 2 assert save_to.read_text().strip() == "hello world" diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live.py b/packages/deepctl-cmd-listen/tests/unit/test_live.py new file mode 100644 index 00000000..2802f4b7 --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_live.py @@ -0,0 +1,780 @@ +"""Tests for the live streaming engine (deepctl_cmd_listen.live).""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import subprocess +import sys +import textwrap +import threading +import time +import types + +import pytest +from deepctl_cmd_listen import live + + +class FakeWS: + """A stand-in server socket. Replays ``frames`` at once and + ``after_close`` once CloseStream arrives, then ends the stream if + ``close_on_closestream`` (like the real API), or when ``finish()`` is + called.""" + + def __init__( + self, frames=(), *, after_close=(), close_on_closestream=True, frame_gap=0.0 + ): + self.sent: list = [] + self.frames = list(frames) + self.after_close = list(after_close) + self.close_on_closestream = close_on_closestream + self.frame_gap = frame_gap + self.got_close = False + self.done = False + self.wake = asyncio.Event() + self.closed = False + self.transport = None + + def finish(self): + self.done = True + self.wake.set() + + async def send(self, data): + self.sent.append(data) + if data == live.CLOSE_STREAM: + self.got_close = True + self.wake.set() + + def __aiter__(self): + return self + + async def __anext__(self): + while True: + pending = self.frames or (self.after_close if self.got_close else []) + if pending: + if self.frame_gap: + await asyncio.sleep(self.frame_gap) + return pending.pop(0) + if self.done or (self.got_close and self.close_on_closestream): + raise StopAsyncIteration + self.wake.clear() + await self.wake.wait() + + async def close(self): + self.closed = True + self.finish() + + +def install_fake_websockets(monkeypatch, ws): + """Make ``websockets.connect(...)`` resolve to ``ws`` (awaitable form).""" + + calls = [] + + class _Connect: + def __await__(self): + async def _open(): + return ws + + return _open().__await__() + + def connect(*args, **kwargs): + calls.append((args, kwargs)) + return _Connect() + + mod = types.ModuleType("websockets") + mod.connect = connect + monkeypatch.setitem(sys.modules, "websockets", mod) + monkeypatch.setattr(live, "_open", connect) + return calls + + +def run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +async def session(ws_messages, *, audio=(), eof=True, stop_after=None, **kw): + """Run run_session with an in-memory source; returns (outcome, messages).""" + loop = asyncio.get_running_loop() + q: asyncio.Queue = asyncio.Queue() + closing = threading.Event() + got: list = [] + + def start(): + async def feed(): + for chunk in audio: + await q.put(chunk) + if eof: + await q.put(b"") + + tasks.append(loop.create_task(feed())) + + tasks: list = [] + stop = asyncio.Event() + if stop_after is not None: + loop.call_later(stop_after, stop.set) + outcome = await live.run_session( + "wss://api.deepgram.com/v1/listen", + {"Authorization": "Token k"}, + on_message=got.append, + queue=q, + closing=closing, + start_source=start, + byte_rate=kw.pop("byte_rate", 32000), + stop=stop, + ) + assert closing.is_set(), "shutdown must always set closing" + return outcome, got + + +def test_bytes_per_second_unknown_is_zero(): + assert live.bytes_per_second("opus", 48000, 1) == 0 + assert live.bytes_per_second("linear16", 0, 1) == 0 + + +# ── Stdin reader ─────────────────────────────────────────────────────────── + + +def test_reader_reads_capped_reads_then_eof(): + r, w = os.pipe() + os.write(w, b"a" * 5000) + os.close(w) + + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=32) + live.StdinReader( + asyncio.get_running_loop(), q, threading.Event(), read_size=1600, fd=r + ).start() + items = [] + while True: + item = await asyncio.wait_for(q.get(), 5) + items.append(item) + if not item: + return items + + items = run(main()) + os.close(r) + assert all(len(i) <= 1600 for i in items) + assert b"".join(items) == b"a" * 5000 + assert items[-1] == b"" + + +def test_reader_stops_without_put_when_closing(): + r, w = os.pipe() + os.write(w, b"x" * 10) + + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=4) + closing = threading.Event() + closing.set() + live.StdinReader( + asyncio.get_running_loop(), q, closing, read_size=100, fd=r + ).start() + await asyncio.sleep(0.1) + return q.qsize() + + assert run(main()) == 0 + os.close(w) + os.close(r) + + +def test_reader_put_after_loop_closed_is_quiet(recwarn): + loop = asyncio.new_event_loop() + q: asyncio.Queue = asyncio.Queue(maxsize=4) + reader = live.StdinReader(loop, q, threading.Event(), read_size=10, fd=0) + loop.close() + assert reader._put(b"x") is False + assert not [w for w in recwarn if "never awaited" in str(w.message)] + + +def test_reader_reports_read_errors(): + r, w = os.pipe() + os.close(w) + os.close(r) # reading a closed descriptor: EBADF + + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=4) + live.StdinReader( + asyncio.get_running_loop(), q, threading.Event(), read_size=10, fd=r + ).start() + return await asyncio.wait_for(q.get(), 5) + + assert isinstance(run(main()), OSError) + + +# Regression: a reader blocked +# handing audio to a slow consumer while Ctrl-C shuts the loop down used to +# abort the interpreter (exit 134, "Fatal Python error") on 3.11/3.12. +_SHUTDOWN_SCRIPT = textwrap.dedent( + """ + import asyncio, sys, threading + from deepctl_cmd_listen import live + + async def main(): + loop = asyncio.get_running_loop() + q = asyncio.Queue(maxsize=32) + closing = threading.Event() + stop = asyncio.Event() + guard = live._SignalGuard(loop, stop) + guard.install() + live.StdinReader(loop, q, closing, read_size=3200).start() + + async def consume(): + while True: + b = await q.get() + if not b: + stop.set() + return + await asyncio.sleep(0.01) # slow: the reader waits on the put + + t = asyncio.create_task(consume()) + print("ready", flush=True) + await stop.wait() + closing.set() # before cancelling, as run_session does + t.cancel() + guard.restore() + + asyncio.run(main()) + print("exited main", file=sys.stderr) + """ +) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") +@pytest.mark.parametrize("attempt", range(5)) +def test_sigint_with_reader_blocked_on_put_exits_cleanly(attempt): + feeder = subprocess.Popen( + [ + sys.executable, + "-c", + "import sys\nwhile True: sys.stdout.buffer.write(b'x'*65536)", + ], + stdout=subprocess.PIPE, + ) + try: + proc = subprocess.Popen( + [sys.executable, "-c", _SHUTDOWN_SCRIPT], + stdin=feeder.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert proc.stdout.readline().strip() == b"ready" + time.sleep(0.3) + proc.send_signal(signal.SIGINT) + _, err = proc.communicate(timeout=10) + finally: + feeder.kill() + feeder.wait() + assert proc.returncode == 0, err.decode() + assert b"Fatal Python error" not in err + assert b"Traceback" not in err + assert b"exited main" in err + + +# ── Session: ends and ordering ───────────────────────────────────────────── + + +def _result(text, start=0.0, duration=1.0): + return json.dumps( + { + "type": "Results", + "is_final": True, + "start": start, + "duration": duration, + "channel": {"alternatives": [{"transcript": text}]}, + } + ) + + +def test_eof_sends_closestream_and_keeps_late_finals(monkeypatch): + # Finals arrive only after CloseStream, like a slow finalize. + ws = FakeWS(after_close=[_result("one"), _result("two")], frame_gap=0.2) + install_fake_websockets(monkeypatch, ws) + outcome, got = run(session([], audio=[b"a" * 100, b"b" * 100])) + assert outcome.status == "success" + assert got == [_result("one"), _result("two")] + assert ws.sent[:2] == [b"a" * 100, b"b" * 100] + assert ws.sent[-1] == live.CLOSE_STREAM + + +def test_server_close_ends_session_cleanly(monkeypatch): + ws = FakeWS([_result("hi")]) + install_fake_websockets(monkeypatch, ws) + + async def main(): + asyncio.get_running_loop().call_later(0.05, ws.finish) + return await session([], eof=False) + + outcome, got = run(main()) + assert outcome.status == "success" + assert got == [_result("hi")] + + +def test_server_error_frame_is_an_error(monkeypatch): + import click + + ws = FakeWS([json.dumps({"type": "Error"})]) + install_fake_websockets(monkeypatch, ws) + + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=4) + + def on_message(raw): + raise click.ClickException("Flux STT error (X): bad") + + return await live.run_session( + "wss://x/v2/listen", + {}, + on_message=on_message, + queue=q, + closing=threading.Event(), + start_source=lambda: None, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.status == "error" + assert "Flux STT error (X): bad" in outcome.error + + +def test_stop_sends_closestream_and_waits_for_server(monkeypatch): + # A 5 s drain cap, so a stop that waited it out is far from the bound. + monkeypatch.setattr(live, "STOP_DRAIN_S", 5.0) + ws = FakeWS([_result("kept")], frame_gap=0.0) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, got = run(session([], audio=[b"a" * 10], eof=False, stop_after=0.1)) + elapsed = time.monotonic() - t0 + assert outcome.status == "cancelled" + assert not outcome.incomplete + assert live.CLOSE_STREAM in ws.sent + assert got == [_result("kept")] + assert elapsed < 3.0 + + +def test_stop_gives_up_after_one_second(monkeypatch): + ws = FakeWS(close_on_closestream=False) # never finishes + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, _ = run(session([], eof=False, stop_after=0.1)) + elapsed = time.monotonic() - t0 + assert outcome.status == "cancelled" + assert outcome.incomplete + assert live.CLOSE_STREAM in ws.sent + assert 1.0 <= elapsed < 1.9 + + +def test_stop_during_eof_drain_waits_the_stop_cap_once(monkeypatch): + # A 3 s stop cap: one wait ends near 3.3 s and two near 6.3 s, so the + # bound is 1.5 s from each and a slow host can't blur them. + monkeypatch.setattr(live, "STOP_DRAIN_S", 3.0) + ws = FakeWS(close_on_closestream=False) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, _ = run(session([], audio=[b"a"], eof=True, stop_after=0.3)) + elapsed = time.monotonic() - t0 + assert outcome.status == "cancelled" + assert outcome.incomplete + assert ws.sent.count(live.CLOSE_STREAM) == 1 # not sent twice + assert 3.2 <= elapsed < 4.8 # the stop cap waited once, not twice + + +def test_eof_idle_deadline(monkeypatch): + monkeypatch.setattr(live, "EOF_IDLE_S", 0.3) + ws = FakeWS(close_on_closestream=False) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, _ = run(session([], audio=[b"a"])) + elapsed = time.monotonic() - t0 + assert outcome.status == "drain_timeout" + assert outcome.drain_reason == "idle" + assert 0.3 <= elapsed < 1.5 + + +def test_eof_idle_deadline_refreshed_by_server_messages(monkeypatch): + monkeypatch.setattr(live, "EOF_IDLE_S", 0.3) + ws = FakeWS( + after_close=[_result(str(i)) for i in range(6)], + close_on_closestream=False, + frame_gap=0.15, + ) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, got = run(session([], audio=[b"a"])) + elapsed = time.monotonic() - t0 + assert len(got) == 6 + assert outcome.status == "drain_timeout" + # 6 x 0.15 s of messages, then 0.3 s idle. + assert 1.1 <= elapsed < 2.6 + + +def test_eof_absolute_cap(monkeypatch): + monkeypatch.setattr(live, "EOF_IDLE_S", 5.0) + monkeypatch.setattr(live, "EOF_CAP_MIN_S", 0.4) + ws = FakeWS( + after_close=[_result(str(i)) for i in range(100)], + close_on_closestream=False, + frame_gap=0.05, + ) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, _ = run(session([], audio=[b"a"])) + elapsed = time.monotonic() - t0 + assert outcome.status == "drain_timeout" + assert outcome.drain_reason == "cap" + assert 0.4 <= elapsed < 2.0 + + +def test_eof_cap_uses_audio_behind_for_pcm(monkeypatch): + # 128 000 B at 320 000 B/s = 0.4 s sent; finals cover 0.1 s; 0.3 s behind + # -> cap = 2 x 0.3 = 0.6 s (with the 30 s floor lowered for the test). + monkeypatch.setattr(live, "EOF_CAP_MIN_S", 0.0) + monkeypatch.setattr(live, "EOF_IDLE_S", 5.0) + ws = FakeWS(after_close=[_result("x", 0.0, 0.1)], close_on_closestream=False) + install_fake_websockets(monkeypatch, ws) + t0 = time.monotonic() + outcome, _ = run(session([], audio=[b"a" * 32000] * 4, byte_rate=320000)) + elapsed = time.monotonic() - t0 + assert outcome.status == "drain_timeout" + assert outcome.drain_reason == "cap" + assert outcome.drain_limit_s == pytest.approx(0.6) + assert 0.6 <= elapsed < 2.0 + + +def test_eof_cap_uses_send_time_without_byte_rate(monkeypatch): + monkeypatch.setattr(live, "EOF_CAP_MIN_S", 0.0) + monkeypatch.setattr(live, "EOF_IDLE_S", 5.0) + ws = FakeWS(close_on_closestream=False) + install_fake_websockets(monkeypatch, ws) + + async def main(): + loop = asyncio.get_running_loop() + q: asyncio.Queue = asyncio.Queue(maxsize=4) + + def start(): + async def feed(): + for _ in range(3): + await q.put(b"o" * 10) + await asyncio.sleep(0.1) + await q.put(b"") + + tasks.append(loop.create_task(feed())) + + tasks: list = [] + return await live.run_session( + "wss://x", + {}, + on_message=lambda m: None, + queue=q, + closing=threading.Event(), + start_source=start, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.drain_reason == "cap" + # about 0.2 s between the first and last send -> cap about 0.4 s + assert 0.4 <= outcome.drain_limit_s <= 2.0 + + +def test_shutdown_sets_closing_before_cancelling(monkeypatch): + """The producer must see ``closing`` before any task is cancelled: the + sender, cancelled while it waits on the queue, records what it saw.""" + ws = FakeWS(close_on_closestream=False) + install_fake_websockets(monkeypatch, ws) + closing = threading.Event() + seen: list = [] + + class SpyQueue(asyncio.Queue): + async def get(self): + try: + return await super().get() + except asyncio.CancelledError: + seen.append(closing.is_set()) + raise + + async def main(): + stop = asyncio.Event() + asyncio.get_running_loop().call_later(0.05, stop.set) + return await live.run_session( + "wss://x", + {}, + on_message=lambda m: None, + queue=SpyQueue(maxsize=4), + closing=closing, + start_source=lambda: None, + byte_rate=0, + stop=stop, + ) + + run(main()) + assert seen == [True] + + +def test_signal_guard_first_stops_second_exits(monkeypatch): + exits = [] + monkeypatch.setattr(live.os, "_exit", lambda code: exits.append(code)) + + async def main(): + stop = asyncio.Event() + guard = live._SignalGuard(asyncio.get_running_loop(), stop) + guard._handler(signal.SIGINT, None) + await asyncio.sleep(0) + first = stop.is_set() + guard._handler(signal.SIGINT, None) + return first + + assert run(main()) is True + assert exits == [2] + + +def test_signal_guard_restores_previous_handler(): + before = signal.getsignal(signal.SIGINT) + + async def main(): + guard = live._SignalGuard(asyncio.get_running_loop(), asyncio.Event()) + guard.install() + installed = signal.getsignal(signal.SIGINT) + guard.restore() + return installed + + installed = run(main()) + assert installed != before + assert signal.getsignal(signal.SIGINT) == before + + +def test_closed_outcomes(): + from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK + from websockets.frames import Close + + ok = ConnectionClosedOK(Close(1000, ""), Close(1000, ""), True) + assert live._closed_outcome(ok, ()) is None + bad = ConnectionClosedError(Close(1011, "fake\x1b[31m error"), None) + out = live._closed_outcome(bad, ()) + assert out.kind == "server" and out.close_code == 1011 + assert "1011" in out.error and "\x1b" not in out.error + lost = live._closed_outcome(ConnectionClosedError(None, None), ()) + assert lost.kind == "network" + assert lost.error == "The connection to the server was lost." + + +# ── Secrets never echoed ──────────────────────────── + +SENTINEL = "fake_SENTINEL_0123456789abcdef" + + +@pytest.mark.parametrize( + "text", + [ + f"rejected Token {SENTINEL}", + f"rejected bearer {SENTINEL}", + f"key {SENTINEL} is bad", + f"{'x' * 190}Token {SENTINEL}", # the cut must not leave a prefix + ], +) +def test_clean_scrubs_credentials_before_truncating(text): + out = live._clean(text, 200, (SENTINEL,)) + assert SENTINEL not in out + assert SENTINEL[:8] not in out + assert len(out) <= 200 + + +def test_server_close_reason_echoing_the_key_is_scrubbed(monkeypatch): + from websockets.exceptions import ConnectionClosedError + from websockets.frames import Close + + class Echo(FakeWS): + async def __anext__(self): + raise ConnectionClosedError( + Close(1011, f"rejected Token {SENTINEL} and {SENTINEL}"), None + ) + + install_fake_websockets(monkeypatch, Echo()) + + async def main(): + return await live.run_session( + "wss://x", + {"Authorization": f"Token {SENTINEL}"}, + on_message=lambda m: None, + queue=asyncio.Queue(), + closing=threading.Event(), + start_source=lambda: None, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.kind == "server" + assert SENTINEL not in outcome.error + + +def test_flux_error_description_echoing_the_key_is_scrubbed(monkeypatch): + import click + + install_fake_websockets(monkeypatch, FakeWS([json.dumps({"type": "Error"})])) + + def on_message(raw): + raise click.ClickException(f"Flux STT error (AUTH): Token {SENTINEL}") + + async def main(): + return await live.run_session( + "wss://x", + {"Authorization": f"Token {SENTINEL}"}, + on_message=on_message, + queue=asyncio.Queue(), + closing=threading.Event(), + start_source=lambda: None, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.kind == "server" + assert SENTINEL not in outcome.error + assert "Flux STT error (AUTH)" in outcome.error + + +# ── stdout closed by its reader ────────────────────────────────────── + + +def test_broken_pipe_on_stdout_ends_quietly(monkeypatch): + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + install_fake_websockets(monkeypatch, FakeWS([_result("a")])) + + def on_message(raw): + raise BrokenPipeError(32, "Broken pipe") + + async def main(): + return await live.run_session( + "wss://x", + {}, + on_message=on_message, + queue=asyncio.Queue(), + closing=threading.Event(), + start_source=lambda: None, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.status == "success" + assert outcome.stdout_closed + assert silenced == [True] + + +# ── SIGTERM and non-blocking stdin ───────────────────────────────────────── + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") +def test_signal_guard_sigterm_drains_then_second_signal_dies_by_sigterm(monkeypatch): + died = [] + monkeypatch.setattr(live, "die_by_signal", lambda s: died.append(s)) + monkeypatch.setattr(live.os, "_exit", lambda code: died.append(("exit", code))) + + async def main(): + stop = asyncio.Event() + guard = live._SignalGuard(asyncio.get_running_loop(), stop) + guard._handler(signal.SIGTERM, None) + await asyncio.sleep(0) + first = (stop.is_set(), guard.signum) + guard._handler(signal.SIGINT, None) + return first + + assert run(main()) == (True, signal.SIGTERM) + assert died[0] == signal.SIGTERM + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") +def test_signal_guard_installs_sigterm_only_during_a_session(): + assert signal.getsignal(signal.SIGTERM) == signal.SIG_DFL + + async def main(): + guard = live._SignalGuard(asyncio.get_running_loop(), asyncio.Event()) + guard.install() + during = signal.getsignal(signal.SIGTERM) + guard.restore() + return during + + assert run(main()) != signal.SIG_DFL + assert signal.getsignal(signal.SIGTERM) == signal.SIG_DFL + + +def test_session_outcome_records_sigterm(monkeypatch): + ws = FakeWS([_result("kept")]) + install_fake_websockets(monkeypatch, ws) + + async def main(): + loop = asyncio.get_running_loop() + loop.call_later(0.1, signal.raise_signal, signal.SIGTERM) + return await live.run_session( + "wss://x", + {}, + on_message=lambda m: None, + queue=asyncio.Queue(), + closing=threading.Event(), + start_source=lambda: None, + byte_rate=0, + ) + + if sys.platform == "win32": + pytest.skip("POSIX signals") + outcome = run(main()) + assert outcome.status == "cancelled" + assert outcome.signal == signal.SIGTERM + assert live.CLOSE_STREAM in ws.sent + assert signal.getsignal(signal.SIGTERM) == signal.SIG_DFL + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") +def test_signal_while_server_closes_still_cancels(monkeypatch): + """A SIGTERM that lands while the server is closing isn't dropped: the + clean close becomes a SIGTERM stop, so the command still re-raises it.""" + + class _SignalOnClose(FakeWS): + async def close(self): + signal.raise_signal(signal.SIGTERM) + await super().close() + + ws = _SignalOnClose([_result("kept")]) + install_fake_websockets(monkeypatch, ws) + + async def main(): + asyncio.get_running_loop().call_later(0.05, ws.finish) + return await session([], eof=False) + + outcome, got = run(main()) + assert got == [_result("kept")] + assert outcome.status == "cancelled" + assert outcome.signal == signal.SIGTERM + assert signal.getsignal(signal.SIGTERM) == signal.SIG_DFL + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX pipes") +def test_reader_waits_on_a_nonblocking_descriptor(): + """A parent may leave O_NONBLOCK on the shared stdin; the reader waits + for data instead of failing with EAGAIN, and leaves the flag alone.""" + r, w = os.pipe() + os.set_blocking(r, False) + + async def main(): + q: asyncio.Queue = asyncio.Queue() + live.StdinReader( + asyncio.get_running_loop(), q, threading.Event(), read_size=100, fd=r + ).start() + await asyncio.sleep(0.3) # nothing to read yet: EAGAIN + os.write(w, b"late") + os.close(w) + items = [] + while True: + item = await asyncio.wait_for(q.get(), 5) + items.append(item) + if not isinstance(item, bytes) or not item: + return items + + items = run(main()) + assert items == [b"late", b""] + assert os.get_blocking(r) is False + os.close(r) diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_command.py b/packages/deepctl-cmd-listen/tests/unit/test_live_command.py new file mode 100644 index 00000000..652b1dbf --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_command.py @@ -0,0 +1,241 @@ +"""ListenCommand's live path: stream failures, SIGTERM, host checks.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest +from deepctl_cmd_listen import live +from deepctl_cmd_listen import messages as msg +from deepctl_cmd_listen.command import ListenCommand +from deepctl_cmd_listen.models import ListenCodedError, ListenResult + +from .test_listen_command import _EndOnCloseStream, _install_websockets, _stdin_pipe + + +@pytest.fixture +def command(): + return ListenCommand() + + +def _client(): + client = MagicMock() + client.config.get_profile.return_value.base_url = "https://api.deepgram.com" + client.auth_manager.get_api_key.return_value = "k" + return client + + +def _results(*texts): + return [ + json.dumps( + { + "type": "Results", + "is_final": True, + "start": 0, + "duration": 1, + "channel": {"alternatives": [{"transcript": t}]}, + } + ) + for t in texts + ] + + +def _stream(command, **over): + kw = { + "model": "nova-3", + "language": "en-US", + "api_version": 1, + "diarize": False, + "smart_format": True, + "punctuate": True, + "interim": True, + "redact": (), + "numerals": False, + "encoding": "linear16", + "sample_rate": 16000, + "channels": 1, + "save_to": None, + "caption_format": None, + } + kw.update(over) + return command._stream_stdin(_client(), **kw) + + +# ── Drain timeout and server errors ──────────────────────────────────────── + + +class _Silent(_EndOnCloseStream): + """Never answers CloseStream.""" + + async def send(self, data): + self.sent.append(data) + + +def test_drain_timeout_is_coded_and_keeps_save_to( + command, monkeypatch, capsys, tmp_path +): + monkeypatch.setattr(live, "EOF_IDLE_S", 0.2) + + class _OneThenSilent(_Silent): + def __init__(self): + super().__init__() + self.frames = _results("kept") + + async def __anext__(self): + if self.frames: + return self.frames.pop(0) + return await super().__anext__() + + _stdin_pipe(monkeypatch, b"\0" * 3200) + _install_websockets(monkeypatch, _OneThenSilent()) + save = tmp_path / "t.txt" + result = _stream(command, save_to=str(save)) + assert isinstance(result, ListenCodedError) + assert result.code == "stream_drain_timeout" + assert result.next_step is None + assert result.details == { + "reason": "idle", + "limit_s": 0.2, + "transcript": "kept", + "transcript_chars": 4, + "saved_to": str(save), + } + assert list(result.model_dump()) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + "details", + ] + assert command.exit_code_for(result) == 1 + assert save.read_text() == "kept" + err = capsys.readouterr().err + assert "after the audio ended, so results may be incomplete" in err + assert msg.H_DRAIN_TIMEOUT in err + + +def test_server_error_result_exits_one(command, monkeypatch, capsys): + import asyncio + + class _Boom(_EndOnCloseStream): + async def __anext__(self): + await asyncio.sleep(0.01) + raise RuntimeError("socket exploded") + + _stdin_pipe(monkeypatch, b"", close=False) + _install_websockets(monkeypatch, _Boom()) + result = _stream(command) + assert isinstance(result, ListenCodedError) + assert result.code == "network_unavailable" + assert result.details["transcript"] == "" + assert result.details["reason"] == "connection lost" + assert command.exit_code_for(result) == 1 + assert "socket exploded" in capsys.readouterr().err + + +def test_untrusted_host_never_connects(command, monkeypatch): + """The WebSocket URL comes only from the checked base URL; a refused + host raises before any connection is attempted.""" + from deepctl_core.hosts import CredentialHostError + + connect = MagicMock() + import sys + import types + + fake = types.ModuleType("websockets") + fake.connect = connect + monkeypatch.setitem(sys.modules, "websockets", fake) + client = _client() + client.auth_manager.check_base_url.side_effect = CredentialHostError( + "E-FH-FLAG", "foreign_host", "no", "hint" + ) + _stdin_pipe(monkeypatch, b"") + with pytest.raises(CredentialHostError): + command._stream_stdin( + client, + model="nova-3", + language="en-US", + api_version=1, + diarize=False, + smart_format=True, + punctuate=True, + interim=False, + redact=(), + numerals=False, + encoding="linear16", + sample_rate=16000, + channels=1, + save_to=None, + caption_format=None, + ) + connect.assert_not_called() + + +# ── A server error close keeps the transcript in details ──────────────────── + + +def test_server_error_close_is_coded_with_transcript(command, monkeypatch, capsys): + from websockets.exceptions import ConnectionClosedError + from websockets.frames import Close + + class _CloseAfter(_EndOnCloseStream): + def __init__(self): + super().__init__() + self.frames = _results("one", "two") + + async def __anext__(self): + if self.frames: + return self.frames.pop(0) + raise ConnectionClosedError(Close(1011, "boom Token k"), None) + + _stdin_pipe(monkeypatch, b"", close=False) + _install_websockets(monkeypatch, _CloseAfter()) + result = _stream(command) + assert result.code == "server_error" + assert result.details["close_code"] == 1011 + assert result.details["transcript"] == "one two" + assert result.details["transcript_chars"] == 7 + assert "saved_to" not in result.details + err = capsys.readouterr().err + assert "(1011)" in err and "Token k" not in err + + +# ── A SIGTERM drain ends the process by SIGTERM ──────────────────────────── + + +def test_exit_code_for_reraises_sigterm_after_drain(command, monkeypatch): + import signal + + died = [] + monkeypatch.setattr(live, "die_by_signal", lambda s: died.append(s)) + command._reraise_signal = signal.SIGTERM + r = ListenResult(status="cancelled", source="stdin", mode="live") + command.exit_code_for(r) + assert died == [signal.SIGTERM] + assert command._reraise_signal is None + + +# ── Round 2: stdin that can't be read is invalid_argument ────────────────── + + +def test_unreadable_stdin_is_invalid_argument(command, monkeypatch, capsys): + import os + + r, w = os.pipe() + os.close(w) + os.close(r) # reading a closed descriptor: EBADF + monkeypatch.setattr("deepctl_cmd_listen.command._stdin_fd", lambda: r) + _install_websockets(monkeypatch, _EndOnCloseStream()) + result = _stream(command) + assert isinstance(result, ListenCodedError) + assert result.code == "invalid_argument" + assert result.error.startswith("Couldn't read audio from stdin: ") + assert result.error.endswith(".") + assert result.next_step.endswith(" listen --help") + assert result.hint in (msg.H_LISTEN_STDIN_UNREADABLE, msg.A_LISTEN_STDIN_UNREADABLE) + assert result.details == {"transcript": "", "transcript_chars": 0} + assert command.exit_code_for(result) == 1 + assert "Couldn't read audio from stdin" in capsys.readouterr().err diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_exit.py b/packages/deepctl-cmd-listen/tests/unit/test_live_exit.py new file mode 100644 index 00000000..4573f565 --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_exit.py @@ -0,0 +1,152 @@ +"""How a live stream's stop maps to its result and exit code, and the +listen command's lazy imports.""" + +from __future__ import annotations + +import sys +from unittest.mock import MagicMock + +import pytest +from deepctl_cmd_listen import live +from deepctl_cmd_listen.command import ListenCommand + +from .test_listen_command import _EndOnCloseStream, _install_websockets + + +@pytest.fixture +def command(): + return ListenCommand() + + +def _client(): + client = MagicMock() + client.config.get_profile.return_value.base_url = "https://api.deepgram.com" + client.auth_manager.get_api_key.return_value = "k" + return client + + +# ── Mic Ctrl-C ends the session (exit 0); other stops stay cancelled ─────── + + +def _finish(command, source, signum): + import signal + + outcome = live.LiveOutcome(status="cancelled", signal=getattr(signal, signum)) + return command._finish_live( + outcome, + source=source, + model="nova-3", + language="en", + diarize=False, + transcript_acc=["hello", "world"], + caption_writer=None, + caption_format=None, + save_to=None, + v2_state=None, + ) + + +@pytest.mark.parametrize( + ("source", "signum", "status", "code"), + [("mic", "SIGINT", "success", 0), ("stdin", "SIGINT", "cancelled", 2)], +) +def test_live_sigint_outcome_by_source(command, source, signum, status, code): + result = _finish(command, source, signum) + assert result.status == status + assert result.transcript == "hello world" + assert command.exit_code_for(result) == code + + +@pytest.mark.skipif(sys.platform == "win32", reason="SIGTERM drain is POSIX only") +@pytest.mark.parametrize("source", ["mic", "stdin"]) +def test_live_sigterm_is_cancelled_for_every_source(command, monkeypatch, source): + import signal + + died = [] + monkeypatch.setattr(live, "die_by_signal", lambda s: died.append(s)) + result = _finish(command, source, "SIGTERM") + assert result.status == "cancelled" + assert result.transcript == "hello world" + command.exit_code_for(result) + assert died == [signal.SIGTERM] + + +def test_mic_sigterm_drains_and_stays_cancelled(command, monkeypatch): + """A real SIGTERM during a mic stream drains like Ctrl-C, but the result + stays "cancelled" and the process then ends by SIGTERM.""" + import asyncio + import signal + import types + + if sys.platform == "win32": + pytest.skip("SIGTERM drain is POSIX only") + + class _FakeInputStream: + def __init__(self, **_kwargs): + pass + + def start(self): + pass + + def stop(self): + pass + + def close(self): + pass + + fake_sd = types.ModuleType("sounddevice") + fake_sd.RawInputStream = _FakeInputStream + fake_np = types.ModuleType("numpy") + fake_np.int16 = "int16" + monkeypatch.setitem(sys.modules, "sounddevice", fake_sd) + monkeypatch.setitem(sys.modules, "numpy", fake_np) + ws = _EndOnCloseStream() + _install_websockets(monkeypatch, ws) + died = [] + monkeypatch.setattr(live, "die_by_signal", lambda s: died.append(s)) + + loop = asyncio.new_event_loop() + loop.call_later(0.1, signal.raise_signal, signal.SIGTERM) + try: + result = loop.run_until_complete( + command._ws_mic( + _client(), + model="nova-3", + language="en", + api_version=1, + diarize=False, + smart_format=True, + punctuate=True, + interim=False, + redact=(), + numerals=False, + sample_rate=16000, + channels=1, + ) + ) + finally: + loop.close() + + assert result.status == "cancelled" + assert '{"type": "CloseStream"}' in ws.sent + command.exit_code_for(result) + assert died == [signal.SIGTERM] + + +# ── Importing the listen command doesn't import asyncio ──────────────────── + + +def test_listen_command_import_is_lazy(): + import subprocess + + code = ( + "import sys\n" + "from deepctl_cmd_listen.command import ListenCommand\n" + "ListenCommand().get_arguments()\n" + "bad = [m for m in ('asyncio', 'deepctl_cmd_listen.live') if m in sys.modules]\n" + "print(','.join(bad))\n" + ) + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.strip() == "" diff --git a/packages/deepctl-core/pyproject.toml b/packages/deepctl-core/pyproject.toml index 84dc1bdd..0b64e905 100644 --- a/packages/deepctl-core/pyproject.toml +++ b/packages/deepctl-core/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "pyyaml>=6.0.0", "platformdirs>=3.0.0,<5", "packaging>=21.0", + "websockets>=13.1,<18", ] [project.optional-dependencies] diff --git a/packages/deepctl-core/src/deepctl_core/websocket.py b/packages/deepctl-core/src/deepctl_core/websocket.py index 78a4c59b..cd4bf4b7 100644 --- a/packages/deepctl-core/src/deepctl_core/websocket.py +++ b/packages/deepctl-core/src/deepctl_core/websocket.py @@ -83,40 +83,61 @@ def _as_refusal( return exc -class _AsyncOpen: - """``async with`` wrapper around websockets' asyncio ``connect``.""" +async def open_async( + url: str, + headers: dict[str, str], + *, + command: str, + payload: str, + **options: Any, +) -> Any: + """Open ``url`` and return the connection, refusing any redirect. - def __init__( - self, url: str, headers: dict[str, str], command: str, payload: str - ) -> None: - from websockets.asyncio.client import connect + Needs websockets 13.0 or later (its asyncio client); the sync variant + works with anything the Deepgram SDK accepts. ``options`` go to + websockets' ``connect`` (``close_timeout``, ``compression`` and so on). - self._url = url - self._command = command - self._payload = payload - refuse = self._refuse + ``command`` and ``payload`` name the command and what it would have + sent, for the error: "dg listen" and "audio", for example. + """ + from websockets.asyncio.client import connect - class _NoRedirect(connect): - # websockets 13.1 and later call this on a failed handshake and - # follow whatever URI it returns; 13.0 never follows. - def process_redirect(self, exc: Exception) -> Exception | str: - result = refuse(exc) - return result if isinstance(result, Exception) else exc + def refuse(exc: BaseException) -> BaseException: + return _as_refusal(url, exc, command=command, payload=payload) - self._connect = _NoRedirect(url, additional_headers=headers) - self._ws: Any = None + class _NoRedirect(connect): + # websockets 13.1 and later call this on a failed handshake and + # follow whatever URI it returns; 13.0 never follows. + def process_redirect(self, exc: Exception) -> Exception | str: + result = refuse(exc) + return result if isinstance(result, Exception) else exc - def _refuse(self, exc: BaseException) -> BaseException: - return _as_refusal(self._url, exc, command=self._command, payload=self._payload) + try: + return await _NoRedirect(url, additional_headers=headers, **options) + except Exception as exc: + refused = refuse(exc) + if refused is exc: + raise + raise refused from exc + + +class _AsyncOpen: + """``async with`` wrapper around :func:`open_async`.""" + + def __init__( + self, + url: str, + headers: dict[str, str], + command: str, + payload: str, + options: dict[str, Any], + ) -> None: + self._args = (url, headers) + self._kwargs = dict(options, command=command, payload=payload) + self._ws: Any = None async def __aenter__(self) -> Any: - try: - self._ws = await self._connect - except Exception as exc: - refused = self._refuse(exc) - if refused is exc: - raise - raise refused from exc + self._ws = await open_async(*self._args, **self._kwargs) return self._ws async def __aexit__(self, *exc_info: Any) -> None: @@ -124,17 +145,15 @@ async def __aexit__(self, *exc_info: Any) -> None: def connect_async( - url: str, headers: dict[str, str], *, command: str, payload: str + url: str, + headers: dict[str, str], + *, + command: str, + payload: str, + **options: Any, ) -> _AsyncOpen: - """Open ``url`` for ``async with``, refusing any redirect. - - Needs websockets 13.0 or later (its asyncio client); the sync variant - works with anything the Deepgram SDK accepts. - - ``command`` and ``payload`` name the command and what it would have - sent, for the error: "dg listen" and "audio", for example. - """ - return _AsyncOpen(url, headers, command, payload) + """Open ``url`` for ``async with``, refusing any redirect (see open_async).""" + return _AsyncOpen(url, headers, command, payload, options) class _SyncOpen: diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index c91c1b67..4db30f83 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -87,15 +87,21 @@ def live_client(monkeypatch): @pytest.fixture -def feed_stdin(monkeypatch): +def feed_stdin(monkeypatch, tmp_path, request): """Return a helper that pipes raw bytes into the listen command's stdin. - The stdin streaming path reads ``sys.stdin.buffer`` (via the listen - module's ``sys``); swap in a BytesIO-backed fake so an in-process run - behaves like ``… | dg listen -``. + The live stdin path reads a file descriptor (``_stdin_fd()``, fd 0 in + production); point it at a file holding the audio so an in-process run + behaves like ``dg listen - < audio.raw``. ``sys.stdin`` is swapped too, + so source auto-detection sees a non-TTY. """ def _feed(pcm: bytes) -> None: + path = tmp_path / "stdin.raw" + path.write_bytes(pcm) + fd = os.open(path, os.O_RDONLY) + request.addfinalizer(lambda: os.close(fd)) + monkeypatch.setattr("deepctl_cmd_listen.command._stdin_fd", lambda: fd) fake_stdin = types.SimpleNamespace( buffer=io.BytesIO(pcm), isatty=lambda: False, diff --git a/uv.lock b/uv.lock index 18f6c53c..83cbfddb 100644 --- a/uv.lock +++ b/uv.lock @@ -1735,6 +1735,8 @@ dependencies = [ { name = "pyjwt" }, { name = "pyyaml" }, { name = "rich" }, + { name = "websockets", version = "16.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "websockets", version = "17.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] [package.optional-dependencies] @@ -1762,6 +1764,7 @@ requires-dist = [ { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.10.0" }, { name = "pyyaml", specifier = ">=6.0.0" }, { name = "rich", specifier = ">=13.0.0,<16" }, + { name = "websockets", specifier = ">=13.1,<18" }, ] provides-extras = ["dev"] From b57d0469b20cee539b377a4797ff5f6b1d5ac913 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:31:01 -0400 Subject: [PATCH 06/18] perf(listen): bound live-stream memory and frame latency - Stdin reads take at most 50 ms of audio, computed from the encoding, sample rate and channels (whole samples; 4 KiB for encodings without a known byte rate). A pipe read returns what is there, so paced input goes up at the writer's pace: 20 ms frames at 16 kHz linear16 and at 8 kHz mulaw, where 4 KiB reads held telephony audio for about 500 ms. - The send queue holds at most 32 buffers. The reader waits for room, so a slow uplink holds back the producer instead of filling memory. - The microphone callback can't wait, so when the queue is full the oldest buffer is dropped and counted; one warning per stream reports how much audio was dropped. Mic blocks are 50 ms. - listen v1 streams send a KeepAlive after 5 s without audio, so a paused producer doesn't time the stream out. - The WebSocket is opened with compression=None: audio doesn't compress, and deflate cost CPU on every frame. --- .../src/deepctl_cmd_listen/command.py | 29 ++-- .../src/deepctl_cmd_listen/live.py | 88 +++++++++- .../src/deepctl_cmd_listen/messages.py | 5 + .../tests/unit/test_live.py | 154 +++++++++++++++++- 4 files changed, 256 insertions(+), 20 deletions(-) diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 252507d7..a46c03b5 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -923,23 +923,19 @@ async def _ws_mic( v2_state = self._new_v2_state() if api_version >= 2 else None loop = asyncio.get_running_loop() - queue: asyncio.Queue[Any] = asyncio.Queue() + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=live.QUEUE_MAX) closing = threading.Event() + buffer = live.DropOldestBuffer(loop, queue, closing) stream: Any = None def callback(indata: Any, frames: int, t: Any, s: Any) -> None: - if closing.is_set(): - return - if s: + if s and not closing.is_set(): print_warning( f"Audio warning: {escape(str(s))}", stderr=True, ignore_quiet=True, ) - try: - loop.call_soon_threadsafe(queue.put_nowait, bytes(indata)) - except RuntimeError: - pass # the event loop already closed + buffer.offer_threadsafe(bytes(indata)) def start_mic() -> None: nonlocal stream @@ -948,7 +944,8 @@ def start_mic() -> None: channels=channels, dtype=np.int16, callback=callback, - blocksize=int(sample_rate * 0.1), + blocksize=live.frame_bytes("linear16", sample_rate, channels) + // (2 * channels), ) stream.start() @@ -970,6 +967,7 @@ def on_message(raw: str | bytes) -> None: queue=queue, closing=closing, start_source=start_mic, + keepalive=api_version < 2, byte_rate=live.bytes_per_second("linear16", sample_rate, channels), stop=early.bind(loop) if early is not None else None, ) @@ -984,6 +982,14 @@ def on_message(raw: str | bytes) -> None: except Exception: pass + if buffer.dropped_buffers: + rate = live.bytes_per_second("linear16", sample_rate, channels) + print_warning( + msg.W_MIC_DROPPED.format( + seconds=f"{buffer.dropped_bytes / rate if rate else 0.0:.1f}" + ), + stderr=True, + ) return self._finish_live( outcome, source="mic", @@ -1115,13 +1121,13 @@ async def _ws_stdin( v2_state = self._new_v2_state() if api_version >= 2 else None loop = asyncio.get_running_loop() - queue: asyncio.Queue[Any] = asyncio.Queue() + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=live.QUEUE_MAX) closing = threading.Event() reader = live.StdinReader( loop, queue, closing, - read_size=live.DEFAULT_READ_BYTES, + read_size=live.frame_bytes(encoding, sample_rate, channels), fd=_stdin_fd(), ) @@ -1142,6 +1148,7 @@ def on_message(raw: str | bytes) -> None: queue=queue, closing=closing, start_source=reader.start, + keepalive=api_version < 2, byte_rate=live.bytes_per_second(encoding, sample_rate, channels), ) return self._finish_live( diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py index 9e7378f2..85f743c0 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py @@ -1,6 +1,6 @@ """Live streaming engine for ``dg listen`` (stdin and microphone). -One WebSocket session: audio goes up from a queue, transcripts come +One WebSocket session: audio goes up from a bounded queue, transcripts come down to a callback, and the session ends one of four ways: - the server closes the stream (exit 0, or 1 if it reported an error); @@ -15,7 +15,8 @@ - the connection fails (exit 1). Stdin is read with ``os.read(0, cap)`` in a daemon thread that hands each -buffer to the event loop and waits until the queue accepts it. The thread is +buffer to the event loop and waits until the bounded queue accepts it, so a +slow uplink holds back the producer instead of filling memory. The thread is never joined. Before every read and every hand-off it checks ``closing``, which shutdown sets before it cancels any task; anything raised in the thread (including a cancelled hand-off while the loop shuts down) ends the thread @@ -45,14 +46,23 @@ # ── Tunables ──────────────────────────────────────────────────────────────── -#: Largest single read from stdin. +#: Each read takes at most this much audio. The plan allows 20-100 ms; 50 ms +#: keeps frames small for latency while a regular file still goes up in a +#: reasonable number of messages. A pipe read returns whatever is there, so +#: paced input goes up at the writer's own pace whatever this is. +FRAME_MS = 50 +#: Read size when the encoding gives no byte rate (opus, flac, containers). DEFAULT_READ_BYTES = 4096 +#: Audio buffers waiting to be sent. With FRAME_MS that is 1.6 s of audio. +QUEUE_MAX = 32 #: Ctrl-C: how long to wait for the server's last results after CloseStream. STOP_DRAIN_S = 1.0 #: End of input: give up when the server has sent nothing for this long. EOF_IDLE_S = 10.0 #: End of input: the absolute cap is max(EOF_CAP_MIN_S, 2 x the work left). EOF_CAP_MIN_S = 30.0 +#: Send a KeepAlive after this long without audio (listen v1 only). +KEEPALIVE_AFTER_S = 5.0 #: Bound on the WebSocket closing handshake once the outcome is known. CLOSE_TIMEOUT_S = 1.0 @@ -60,6 +70,7 @@ _PCM_WIDTH = {"linear16": 2, "linear32": 4, "mulaw": 1, "alaw": 1} CLOSE_STREAM = json.dumps({"type": "CloseStream"}) +KEEP_ALIVE = json.dumps({"type": "KeepAlive"}) def bytes_per_second(encoding: str | None, sample_rate: int, channels: int) -> int: @@ -70,11 +81,24 @@ def bytes_per_second(encoding: str | None, sample_rate: int, channels: int) -> i return width * sample_rate * channels +def frame_bytes(encoding: str | None, sample_rate: int, channels: int) -> int: + """Largest read for one frame: FRAME_MS of audio, or a fixed size. + + Rounded down to whole sample frames so a read never splits a sample. + """ + rate = bytes_per_second(encoding, sample_rate, channels) + if not rate: + return DEFAULT_READ_BYTES + width = _PCM_WIDTH[(encoding or "").lower()] * channels + size = rate * FRAME_MS // 1000 + return max(width, size - size % width) + + # ── Audio sources ─────────────────────────────────────────────────────────── class StdinReader: - """Reads a file descriptor in a daemon thread into a queue. + """Reads a file descriptor in a daemon thread into a bounded queue. Puts ``bytes`` for audio, ``b""`` once at end of input, or the ``OSError`` a read raised. Never joined: a read blocked on an idle pipe @@ -154,6 +178,45 @@ def _wait_readable(fd: int) -> None: time.sleep(0.2) +class DropOldestBuffer: + """Microphone hand-off: the audio callback can't block, so when the queue + is full the oldest buffer goes. Counts what it dropped.""" + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + queue: asyncio.Queue[Any], + closing: threading.Event, + ) -> None: + self._loop = loop + self._queue = queue + self._closing = closing + self.dropped_buffers = 0 + self.dropped_bytes = 0 + + def offer(self, data: bytes) -> None: + """Call on the event loop: enqueue ``data``, dropping the oldest.""" + if self._queue.full(): + try: + old = self._queue.get_nowait() + except asyncio.QueueEmpty: + pass + else: + self.dropped_buffers += 1 + if isinstance(old, (bytes, bytearray)): + self.dropped_bytes += len(old) + self._queue.put_nowait(data) + + def offer_threadsafe(self, data: bytes) -> None: + """Call from the audio thread; never raises, never blocks.""" + if self._closing.is_set(): + return + try: + self._loop.call_soon_threadsafe(self.offer, data) + except BaseException: + pass + + # ── Session ───────────────────────────────────────────────────────────────── @@ -444,6 +507,7 @@ async def run_session( queue: asyncio.Queue[Any], closing: threading.Event, start_source: Callable[[], None], + keepalive: bool, byte_rate: int, stop: asyncio.Event | None = None, ) -> LiveOutcome: @@ -528,6 +592,15 @@ async def send_audio() -> str: await ws.send(CLOSE_STREAM) return "eof" + async def keep_alive() -> None: + last = now() + while True: + await asyncio.sleep(0.5) + t = now() + if t - max(clock.last_audio, last) >= KEEPALIVE_AFTER_S: + await ws.send(KEEP_ALIVE) + last = t + def eof_cap() -> float: if byte_rate: behind = clock.bytes_sent / byte_rate - clock.last_final_end @@ -539,7 +612,7 @@ def eof_cap() -> float: async def finish_on_stop(deadline: float) -> LiveOutcome: """Ctrl-C: stop sending, CloseStream, wait for the server until deadline.""" closing.set() - await _cancel_all(tasks.get("send")) + await _cancel_all(tasks.get("send"), tasks.get("keepalive")) recv = tasks["recv"] assert recv is not None if clock.eof_at is None and not recv.done(): @@ -563,7 +636,7 @@ async def finish_on_stop(deadline: float) -> LiveOutcome: stop_wait = asyncio.ensure_future(stop.wait()) tasks["stop"] = stop_wait connect = asyncio.ensure_future( - _open(url, headers, close_timeout=CLOSE_TIMEOUT_S) + _open(url, headers, compression=None, close_timeout=CLOSE_TIMEOUT_S) ) tasks["connect"] = connect await asyncio.wait({connect, stop_wait}, return_when=asyncio.FIRST_COMPLETED) @@ -578,6 +651,8 @@ async def finish_on_stop(deadline: float) -> LiveOutcome: recv = asyncio.ensure_future(receive()) send = asyncio.ensure_future(send_audio()) tasks.update(recv=recv, send=send) + if keepalive: + tasks["keepalive"] = asyncio.ensure_future(keep_alive()) # ── Streaming ───────────────────────────────────────────────── await asyncio.wait({recv, send, stop_wait}, return_when=asyncio.FIRST_COMPLETED) @@ -611,6 +686,7 @@ async def finish_on_stop(deadline: float) -> LiveOutcome: ) # ── End of input: drain ─────────────────────────────────────── + await _cancel_all(tasks.get("keepalive")) assert clock.eof_at is not None cap = eof_cap() cap_at = clock.eof_at + cap diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py index 6011ac93..d5488894 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py @@ -57,6 +57,11 @@ "so the transcript may be incomplete." ) +# Microphone: audio dropped because sending fell behind. Once per stream. +W_MIC_DROPPED = ( + "Dropped {seconds} s of microphone audio because the connection couldn't keep up." +) + # server_error: the server closed the stream with an error close code. E_LISTEN_SERVER_CLOSED = ( "The server ended the stream with an error ({close_code}): {close_reason}" diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live.py b/packages/deepctl-cmd-listen/tests/unit/test_live.py index 2802f4b7..f4a1e09e 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_live.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_live.py @@ -101,7 +101,7 @@ def run(coro): async def session(ws_messages, *, audio=(), eof=True, stop_after=None, **kw): """Run run_session with an in-memory source; returns (outcome, messages).""" loop = asyncio.get_running_loop() - q: asyncio.Queue = asyncio.Queue() + q: asyncio.Queue = asyncio.Queue(maxsize=live.QUEUE_MAX) closing = threading.Event() got: list = [] @@ -125,6 +125,7 @@ async def feed(): queue=q, closing=closing, start_source=start, + keepalive=kw.pop("keepalive", True), byte_rate=kw.pop("byte_rate", 32000), stop=stop, ) @@ -132,15 +133,93 @@ async def feed(): return outcome, got +# ── Frame sizing ──────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("encoding", "rate", "channels", "expected"), + [ + ("linear16", 16000, 1, 1600), # 50 ms + ("linear16", 8000, 1, 800), + ("linear16", 48000, 2, 9600), + ("mulaw", 8000, 1, 400), + ("alaw", 8000, 1, 400), + ("linear32", 16000, 1, 3200), + ("LINEAR16", 16000, 1, 1600), + ("opus", 48000, 1, live.DEFAULT_READ_BYTES), + ("flac", 16000, 1, live.DEFAULT_READ_BYTES), + (None, 16000, 1, live.DEFAULT_READ_BYTES), + ], +) +def test_frame_bytes(encoding, rate, channels, expected): + assert live.frame_bytes(encoding, rate, channels) == expected + + +@pytest.mark.parametrize( + ("encoding", "rate", "channels"), + [ + ("linear16", 16000, 1), + ("linear16", 11025, 3), + ("mulaw", 8000, 1), + ("linear32", 22050, 2), + ], +) +def test_frame_bytes_is_20_to_100_ms_of_whole_samples(encoding, rate, channels): + size = live.frame_bytes(encoding, rate, channels) + bps = live.bytes_per_second(encoding, rate, channels) + assert 0.020 <= size / bps <= 0.100 + width = {"linear16": 2, "linear32": 4, "mulaw": 1}[encoding] * channels + assert size % width == 0 + + def test_bytes_per_second_unknown_is_zero(): assert live.bytes_per_second("opus", 48000, 1) == 0 assert live.bytes_per_second("linear16", 0, 1) == 0 +# ── Drop-oldest (microphone) ─────────────────────────────────────────────── + + +def test_drop_oldest_keeps_newest_and_counts(): + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=3) + buf = live.DropOldestBuffer(asyncio.get_running_loop(), q, threading.Event()) + for i in range(5): + buf.offer(bytes([i]) * 10) + kept = [q.get_nowait() for _ in range(q.qsize())] + return buf, kept + + buf, kept = run(main()) + assert kept == [b"\x02" * 10, b"\x03" * 10, b"\x04" * 10] + assert buf.dropped_buffers == 2 + assert buf.dropped_bytes == 20 + + +def test_drop_oldest_threadsafe_never_raises_after_close(): + loop = asyncio.new_event_loop() + q: asyncio.Queue = asyncio.Queue(maxsize=2) + buf = live.DropOldestBuffer(loop, q, threading.Event()) + loop.close() + buf.offer_threadsafe(b"x") # loop closed: silently ignored + + +def test_drop_oldest_ignores_audio_once_closing(): + async def main(): + q: asyncio.Queue = asyncio.Queue(maxsize=2) + closing = threading.Event() + closing.set() + buf = live.DropOldestBuffer(asyncio.get_running_loop(), q, closing) + buf.offer_threadsafe(b"x") + await asyncio.sleep(0.01) + return q.qsize() + + assert run(main()) == 0 + + # ── Stdin reader ─────────────────────────────────────────────────────────── -def test_reader_reads_capped_reads_then_eof(): +def test_reader_reads_capped_frames_then_eof(): r, w = os.pipe() os.write(w, b"a" * 5000) os.close(w) @@ -164,6 +243,33 @@ async def main(): assert items[-1] == b"" +def test_reader_blocks_on_full_queue_instead_of_buffering(): + """Backpressure: with nobody consuming, the reader holds at most + maxsize buffers plus the one it is handing over.""" + r, w = os.pipe() + os.set_blocking(w, False) + written = 0 + + async def main(): + nonlocal written + q: asyncio.Queue = asyncio.Queue(maxsize=4) + live.StdinReader( + asyncio.get_running_loop(), q, threading.Event(), read_size=100, fd=r + ).start() + for _ in range(200): + try: + written += os.write(w, b"z" * 100) + except BlockingIOError: + break + await asyncio.sleep(0.001) + await asyncio.sleep(0.2) + return q.qsize() + + assert run(main()) == 4 + os.close(w) + os.close(r) + + def test_reader_stops_without_put_when_closing(): r, w = os.pipe() os.write(w, b"x" * 10) @@ -217,7 +323,7 @@ async def main(): async def main(): loop = asyncio.get_running_loop() - q = asyncio.Queue(maxsize=32) + q = asyncio.Queue(maxsize=live.QUEUE_MAX) closing = threading.Event() stop = asyncio.Event() guard = live._SignalGuard(loop, stop) @@ -302,6 +408,15 @@ def test_eof_sends_closestream_and_keeps_late_finals(monkeypatch): assert ws.sent[-1] == live.CLOSE_STREAM +def test_connect_uses_no_compression(monkeypatch): + ws = FakeWS() + calls = install_fake_websockets(monkeypatch, ws) + run(session([])) + ((args, kwargs),) = calls + assert kwargs["compression"] is None + assert args[1] == {"Authorization": "Token k"} + + def test_server_close_ends_session_cleanly(monkeypatch): ws = FakeWS([_result("hi")]) install_fake_websockets(monkeypatch, ws) @@ -334,6 +449,7 @@ def on_message(raw): queue=q, closing=threading.Event(), start_source=lambda: None, + keepalive=False, byte_rate=0, ) @@ -473,6 +589,7 @@ async def feed(): queue=q, closing=threading.Event(), start_source=start, + keepalive=False, byte_rate=0, ) @@ -482,6 +599,32 @@ async def feed(): assert 0.4 <= outcome.drain_limit_s <= 2.0 +def test_keepalive_after_silence(monkeypatch): + monkeypatch.setattr(live, "KEEPALIVE_AFTER_S", 0.2) + ws = FakeWS() + install_fake_websockets(monkeypatch, ws) + + async def main(): + asyncio.get_running_loop().call_later(1.3, ws.finish) + return await session([], eof=False) + + run(main()) + assert ws.sent.count(live.KEEP_ALIVE) >= 2 + + +def test_no_keepalive_when_disabled(monkeypatch): + monkeypatch.setattr(live, "KEEPALIVE_AFTER_S", 0.1) + ws = FakeWS() + install_fake_websockets(monkeypatch, ws) + + async def main(): + asyncio.get_running_loop().call_later(0.5, ws.finish) + return await session([], eof=False, keepalive=False) + + run(main()) + assert live.KEEP_ALIVE not in ws.sent + + def test_shutdown_sets_closing_before_cancelling(monkeypatch): """The producer must see ``closing`` before any task is cancelled: the sender, cancelled while it waits on the queue, records what it saw.""" @@ -508,6 +651,7 @@ async def main(): queue=SpyQueue(maxsize=4), closing=closing, start_source=lambda: None, + keepalive=True, byte_rate=0, stop=stop, ) @@ -604,6 +748,7 @@ async def main(): queue=asyncio.Queue(), closing=threading.Event(), start_source=lambda: None, + keepalive=False, byte_rate=0, ) @@ -628,6 +773,7 @@ async def main(): queue=asyncio.Queue(), closing=threading.Event(), start_source=lambda: None, + keepalive=False, byte_rate=0, ) @@ -656,6 +802,7 @@ async def main(): queue=asyncio.Queue(), closing=threading.Event(), start_source=lambda: None, + keepalive=False, byte_rate=0, ) @@ -716,6 +863,7 @@ async def main(): queue=asyncio.Queue(), closing=threading.Event(), start_source=lambda: None, + keepalive=False, byte_rate=0, ) From 2735cd9d94aede1e2f74b9e79fdda567b6390414 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:31:01 -0400 Subject: [PATCH 07/18] fix(listen): keep machine output to the final result When JSON or YAML was asked for (-o json, -o yaml) or chosen because an AI-agent marker is set, a live stream writes no transcript lines, interim rewrites or live captions to stdout. The final Result object is the one document on stdout, whatever the stream's status, so `dg -o json listen < call.raw | jq` parses. With --verbose, final lines go to stderr as INFO: lines. --save-to still writes the transcript or captions. JSON implied only by the TTY/CI heuristics (cron, CI, a container with no TERM) keeps 0.4.0's live output: lines as they arrive, then the final Result, because those pipelines read stdout as a stream. To tell the cases apart, deepctl_core.output gains format_source(): "explicit", "hard_agent", "soft" or "default". Human output is unchanged. Under a hard AI-agent signal, `dg listen` with no SOURCE and no --mic is refused before authentication with invalid_argument (E-LISTEN-NO-SOURCE, exit 1) instead of streaming a non-TTY stdin until an end of input that may never come. `dg listen -` reads stdin explicitly. CI and people keep auto-detection. --- .../src/deepctl_cmd_listen/captions.py | 8 +- .../src/deepctl_cmd_listen/command.py | 116 ++++++-- .../src/deepctl_cmd_listen/messages.py | 8 + .../tests/unit/test_live_command.py | 257 +++++++++++++++- .../tests/unit/test_live_signals.py | 281 ++++++++++++++++++ .../deepctl-core/src/deepctl_core/output.py | 23 ++ 6 files changed, 668 insertions(+), 25 deletions(-) create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_live_signals.py diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py index 355257a5..bfe1980c 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/captions.py @@ -95,8 +95,10 @@ class StreamingCaptionWriter: word-level speaker data is present. """ - def __init__(self, fmt: str) -> None: + def __init__(self, fmt: str, *, echo: bool = True) -> None: self.fmt = fmt # "webvtt" | "srt" + # False under a machine format: collect words, print nothing. + self.echo = echo self._count = 0 self._all_words: list[dict[str, Any]] = [] @@ -106,7 +108,7 @@ def print_header(self, note: str | None = None) -> None: For WebVTT, an optional NOTE block can carry metadata (model, language, etc.) in a spec-compliant way. SRT has no header equivalent. """ - if self.fmt == "webvtt": + if self.echo and self.fmt == "webvtt": print("WEBVTT", flush=True) print(flush=True) if note: @@ -132,6 +134,8 @@ def write_entry( self._count += 1 self._all_words.extend(words) + if not self.echo: + return text = " ".join(w.get("punctuated_word") or w.get("word", "") for w in words) first_speaker: int | None = words[0].get("speaker") if words else None diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index a46c03b5..01a48ed7 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -33,9 +33,12 @@ from deepctl_core.hosts import CredentialHostError from deepctl_core.output import ( _agentic, + format_source, + get_output_format, is_ai_agent, print_error, print_hint, + print_info, print_warning, ) from rich.console import Console @@ -106,6 +109,18 @@ def __getattr__(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +def _machine_format() -> bool: + """True when live transcript lines stay off stdout: a machine format + that was asked for (``-o json``/``-o yaml``) or chosen because of a hard + AI-agent signal. The final result is then the only payload. + JSON implied by soft detection alone (CI, cron, no TERM) keeps 0.4.0's + live lines, since those pipelines read stdout as a stream.""" + return get_output_format() in ("json", "yaml") and format_source() in ( + "explicit", + "hard_agent", + ) + + _freeze_registered = False @@ -165,6 +180,10 @@ class ListenCommand(BaseCommand): requires_project = False ci_friendly = True + #: Live transcript lines go to stdout; False under a machine format. + _echo: bool = True + #: ``--verbose``: under a machine format, finals go to stderr as INFO. + _verbose: bool = False #: Host of the live socket, for network_unavailable details. _ws_host: str = "" #: Set after a SIGTERM drain; exit_code_for re-raises it. @@ -361,6 +380,28 @@ def get_arguments(self) -> list[dict[str, Any]]: # ── Entry point ──────────────────────────────────────────────────── + def pre_auth_check(self, config: Config, **kwargs: Any) -> Any | None: + """Under a hard AI-agent signal, refuse a run with no source. + + Without SOURCE or --mic, a non-TTY stdin would be streamed until an + end of input that may never come. Agents name stdin with ``-`` + explicitly. Runs before authentication, so nothing is sent. + """ + if kwargs.get("source") is not None or kwargs.get("mic"): + return None + if not is_ai_agent(): + return None + prog = _prog() + text = msg.E_LISTEN_NO_SOURCE.format(prog=prog) + print_error(escape(text)) + print_hint(escape(msg.H_LISTEN_NO_SOURCE)) + return ListenCodedError( + error=text, + code=msg.CODE_INVALID_ARGUMENT, + hint=msg.H_LISTEN_NO_SOURCE, + next_step=f"{prog} {self.name} --help", + ) + def handle( self, config: Config, @@ -371,6 +412,7 @@ def handle( source: str | None = kwargs.get("source") use_mic: bool = kwargs.get("mic", False) guided_flow = False + self._verbose = config.get("output.verbose", False) is True # ── Resolve mode ─────────────────────────────────────────────── if source == "-": @@ -808,6 +850,7 @@ def _stream_mic( from . import live + self._echo = not _machine_format() # From here on Ctrl-C ends the mic session, even before the # session's own handler is in place (imports, connecting). early = live.EarlyStop() @@ -815,7 +858,7 @@ def _stream_mic( caption_writer: StreamingCaptionWriter | None = None if caption_format: - caption_writer = StreamingCaptionWriter(caption_format) + caption_writer = StreamingCaptionWriter(caption_format, echo=self._echo) note = f"Model: {model} · Language: {language}" if diarize: note += " · Speaker labels enabled" @@ -1026,6 +1069,7 @@ def _stream_stdin( from . import live + self._echo = not _machine_format() resolved_encoding = encoding or "linear16" if not encoding: print_warning( @@ -1038,7 +1082,7 @@ def _stream_stdin( caption_writer: StreamingCaptionWriter | None = None if caption_format: - caption_writer = StreamingCaptionWriter(caption_format) + caption_writer = StreamingCaptionWriter(caption_format, echo=self._echo) note = f"Model: {model} · Language: {language}" if diarize: note += " · Speaker labels enabled" @@ -1448,18 +1492,32 @@ def _handle_ws_message( line = format_diarized_words(words) if line: transcript_acc.append(line) - print(line, flush=True) + self._print_final(line) return transcript = alt.get("transcript", "") if transcript: transcript_acc.append(transcript) - print(transcript, flush=True) + self._print_final(transcript) elif interim and not caption_writer: transcript = alt.get("transcript", "") if transcript: - print(f"\r{transcript} ", end="", flush=True) + self._print_interim(transcript) + + def _print_final(self, line: str) -> None: + """A final transcript line: stdout, or under a machine format only + ``INFO:`` on stderr with ``--verbose`` (the result is the payload).""" + if self._echo: + print(line, flush=True) + else: + if self._verbose: + print_info(escape(line), stderr=True) + + def _print_interim(self, text: str) -> None: + """An interim rewrite; dropped under a machine format.""" + if self._echo: + print(f"\r{text} ", end="", flush=True) @staticmethod def _new_v2_state() -> dict[str, Any]: @@ -1514,7 +1572,7 @@ def _handle_v2_turn( if event == "EndOfTurn": self._emit_v2_turn(st, transcript_acc, caption_writer=caption_writer) elif event == "Update" and interim and not caption_writer and transcript: - print(f"\r{transcript} ", end="", flush=True) + self._print_interim(transcript) def _emit_v2_turn( self, @@ -1537,7 +1595,7 @@ def _emit_v2_turn( transcript_acc.append(transcript) else: transcript_acc.append(transcript) - print(transcript, flush=True) + self._print_final(transcript) @staticmethod def _timed_v2_words( @@ -1593,29 +1651,45 @@ def output_result(self, result: Any, config: Config) -> None: import json as _json from datetime import date, datetime + from . import live + def _default(obj: Any) -> str: if isinstance(obj, (datetime, date)): return obj.isoformat() return str(obj) - from deepctl_core.output import get_output_format - fmt = get_output_format() - if not isinstance(result, ListenResult) or result.status != "success": - super().output_result(result, config) + # Live: transcript lines (or captions) were printed as they arrived. + # Under a machine format nothing was, and the final result, whatever + # its status, is the one payload on stdout. + if isinstance(result, ListenResult) and result.mode == "live": + if fmt not in ("json", "yaml"): + return + payload = result.model_dump(exclude_none=True) + try: + if fmt == "json": + sys.stdout.write( + _json.dumps( + payload, indent=2, ensure_ascii=False, default=_default + ) + + "\n" + ) + sys.stdout.flush() + else: + import yaml + + sys.stdout.write( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True) + ) + sys.stdout.flush() + except BrokenPipeError: + # The reader went away (`| true`): end quietly, exit as is. + live.silence_stdout() return - # Live results are printed in real-time; only show a brief summary - if result.mode == "live": - if result.caption_format: - return # captions were already printed incrementally - if result.status == "success" and not result.transcript: - return # nothing to add - if fmt == "json": - out.print_json( - _json.dumps(result.model_dump(exclude_none=True), default=_default) - ) + if not isinstance(result, ListenResult) or result.status != "success": + super().output_result(result, config) return # Caption mode: print captions to stdout regardless of output format diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py index d5488894..f8d51f3e 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py @@ -13,6 +13,14 @@ CODE_SERVER_ERROR = "server_error" CODE_NETWORK_UNAVAILABLE = "network_unavailable" +# invalid_argument: a hard agent signal, no SOURCE, no --mic. +E_LISTEN_NO_SOURCE = ( + "{prog} listen needs a file, a URL, --mic, or - to read audio from stdin." +) +H_LISTEN_NO_SOURCE = ( + "Pass a file path or URL. To stream audio from stdin, pass - explicitly." +) + # invalid_argument: reading stdin failed. E_LISTEN_STDIN_UNREADABLE = "Couldn't read audio from stdin: {os_error}." H_LISTEN_STDIN_UNREADABLE = ( diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_command.py b/packages/deepctl-cmd-listen/tests/unit/test_live_command.py index 652b1dbf..f7ec239f 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_live_command.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_command.py @@ -1,15 +1,18 @@ -"""ListenCommand's live path: stream failures, SIGTERM, host checks.""" +"""ListenCommand's live surface: no-source refusal, machine output, drain timeout.""" from __future__ import annotations import json -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock, patch import pytest +from click.testing import CliRunner from deepctl_cmd_listen import live from deepctl_cmd_listen import messages as msg +from deepctl_cmd_listen.captions import StreamingCaptionWriter from deepctl_cmd_listen.command import ListenCommand from deepctl_cmd_listen.models import ListenCodedError, ListenResult +from deepctl_core import Config, PluginManager from .test_listen_command import _EndOnCloseStream, _install_websockets, _stdin_pipe @@ -19,6 +22,22 @@ def command(): return ListenCommand() +@pytest.fixture +def no_agent(monkeypatch): + for name in ( + "AI_AGENT", + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CODEX_SANDBOX", + "CODEX_SANDBOX_NETWORK_DISABLED", + "CURSOR_AGENT", + "GEMINI_CLI", + "OR_APP_NAME", + "OR_SITE_URL", + ): + monkeypatch.delenv(name, raising=False) + + def _client(): client = MagicMock() client.config.get_profile.return_value.base_url = "https://api.deepgram.com" @@ -26,6 +45,82 @@ def _client(): return client +def _invoke(command, args, *, env=None): + config = Mock(spec=Config) + config.get.side_effect = lambda key, default=None: ( + True if key == "output.quiet" else default + ) + click_command = PluginManager()._create_click_command(command) + with ( + patch("deepctl_core.base_command.AuthManager.guard") as guard, + patch( + "deepctl_core.base_command.AuthManager.check_base_url", + return_value="https://api.deepgram.com", + ), + ): + result = CliRunner().invoke( + click_command, args, obj={"config": config}, env=env, catch_exceptions=False + ) + return result, guard + + +# ── No source under a hard agent signal ──────────────────────────────────── + + +def test_agent_without_source_is_refused_before_auth(command, monkeypatch, no_agent): + monkeypatch.setenv("AI_AGENT", "x") + connect = MagicMock() + _install_websockets(monkeypatch, connect) + result, guard = _invoke(command, []) + assert result.exit_code == 1 + guard.assert_not_called() # nothing authenticated, nothing sent + assert ( + "listen needs a file, a URL, --mic, or - to read audio from stdin." + in result.stderr + ) + assert msg.H_LISTEN_NO_SOURCE in result.stderr + + +def test_agent_no_source_payload(command, monkeypatch, no_agent): + monkeypatch.setenv("AI_AGENT", "x") + early = command.pre_auth_check(Mock(spec=Config), source=None, mic=False) + assert isinstance(early, ListenCodedError) + data = early.model_dump() + assert list(data) == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + ] + assert data["status"] == "error" + assert data["code"] == "invalid_argument" + assert data["next_step"].endswith(" listen --help") + assert data["schema_version"] == 1 + assert command.exit_code_for(early) == 1 + + +@pytest.mark.parametrize( + "kwargs", [{"source": "-"}, {"source": "a.wav"}, {"mic": True}] +) +def test_agent_with_a_source_is_not_refused(command, monkeypatch, no_agent, kwargs): + monkeypatch.setenv("AI_AGENT", "x") + assert command.pre_auth_check(Mock(spec=Config), **kwargs) is None + + +def test_no_agent_keeps_auto_detection(command, no_agent): + assert command.pre_auth_check(Mock(spec=Config), source=None, mic=False) is None + + +def test_ci_alone_is_not_a_hard_signal(command, monkeypatch, no_agent): + monkeypatch.setenv("CI", "true") + assert command.pre_auth_check(Mock(spec=Config), source=None, mic=False) is None + + +# ── A machine format keeps transcript lines off stdout ───────────────────── + + def _results(*texts): return [ json.dumps( @@ -41,6 +136,23 @@ def _results(*texts): ] +class _Replay(_EndOnCloseStream): + def __init__(self, frames): + super().__init__() + self.frames = list(frames) + + async def __anext__(self): + if self.frames: + return self.frames.pop(0) + return await super().__anext__() + + +def _fmt(monkeypatch, fmt, source="explicit"): + monkeypatch.setattr("deepctl_cmd_listen.command.get_output_format", lambda: fmt) + monkeypatch.setattr("deepctl_core.output.get_output_format", lambda: fmt) + monkeypatch.setattr("deepctl_cmd_listen.command.format_source", lambda: source) + + def _stream(command, **over): kw = { "model": "nova-3", @@ -62,6 +174,76 @@ def _stream(command, **over): return command._stream_stdin(_client(), **kw) +def test_machine_format_prints_only_the_result(command, monkeypatch, capsys): + _fmt(monkeypatch, "json") + _stdin_pipe(monkeypatch, b"\0" * 3200) + _install_websockets(monkeypatch, _Replay(_results("hello", "world"))) + result = _stream(command) + assert result.status == "success" + assert capsys.readouterr().out == "" # nothing streamed + command.output_result(result, Mock(spec=Config)) + out = capsys.readouterr().out + payload = json.loads(out) # exactly one document + assert payload["transcript"] == "hello world" + assert payload["status"] == "success" + assert payload["mode"] == "live" + + +def test_human_format_streams_lines(command, monkeypatch, capsys): + _fmt(monkeypatch, "default", "default") + _stdin_pipe(monkeypatch, b"\0" * 3200) + _install_websockets(monkeypatch, _Replay(_results("hello", "world"))) + result = _stream(command) + command.output_result(result, Mock(spec=Config)) + assert capsys.readouterr().out == "hello\nworld\n" + + +def test_machine_format_result_on_empty_stdin(command, monkeypatch, capsys): + """/dev/null on stdin: still exactly one Result document.""" + _fmt(monkeypatch, "json") + _stdin_pipe(monkeypatch, b"") + _install_websockets(monkeypatch, _Replay([])) + result = _stream(command) + command.output_result(result, Mock(spec=Config)) + payload = json.loads(capsys.readouterr().out) + assert payload["transcript"] == "" + + +def test_machine_format_captions_not_streamed(command, monkeypatch, capsys, tmp_path): + _fmt(monkeypatch, "json") + _stdin_pipe(monkeypatch, b"\0" * 3200) + frame = json.dumps( + { + "type": "Results", + "is_final": True, + "start": 0, + "duration": 1, + "channel": { + "alternatives": [ + { + "transcript": "hi", + "words": [{"word": "hi", "start": 0.0, "end": 0.5}], + } + ] + }, + } + ) + _install_websockets(monkeypatch, _Replay([frame])) + save = tmp_path / "c.vtt" + result = _stream(command, caption_format="webvtt", save_to=str(save)) + assert capsys.readouterr().out == "" + assert "WEBVTT" in save.read_text() + assert result.caption_format == "webvtt" + + +def test_caption_writer_echo_false_collects_silently(capsys): + w = StreamingCaptionWriter("srt", echo=False) + w.print_header() + w.write_entry([{"word": "a", "start": 0.0, "end": 1.0}], 0.0, 1.0) + assert capsys.readouterr().out == "" + assert w.accumulated_words == [{"word": "a", "start": 0.0, "end": 1.0}] + + # ── Drain timeout and server errors ──────────────────────────────────────── @@ -174,6 +356,15 @@ def test_untrusted_host_never_connects(command, monkeypatch): connect.assert_not_called() +def test_cancelled_live_result_is_payload_under_json(command, monkeypatch, capsys): + _fmt(monkeypatch, "json") + r = ListenResult(status="cancelled", source="stdin", mode="live", transcript="x") + command.output_result(r, Mock(spec=Config)) + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "cancelled" + assert payload["transcript"] == "x" + + # ── A server error close keeps the transcript in details ──────────────────── @@ -203,6 +394,50 @@ async def __anext__(self): assert "(1011)" in err and "Token k" not in err +# ── Implicit JSON from soft signals only keeps live lines ────────────────── + + +@pytest.mark.parametrize( + ("source", "lines"), + [("soft", True), ("explicit", False), ("hard_agent", False)], +) +def test_live_lines_follow_format_source(command, monkeypatch, capsys, source, lines): + _fmt(monkeypatch, "json", source) + _stdin_pipe(monkeypatch, b"\0" * 3200) + _install_websockets(monkeypatch, _Replay(_results("hello", "world"))) + result = _stream(command) + streamed = capsys.readouterr().out + assert (streamed == "hello\nworld\n") is lines + command.output_result(result, Mock(spec=Config)) + payload = json.loads(capsys.readouterr().out) # then one Result document + assert payload["transcript"] == "hello world" + + +def test_format_source_values(monkeypatch): + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "agentic", False) + output.setup_output("default") + assert output.format_source() == "default" + output.setup_output("json") + assert output.format_source() == "explicit" + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.delenv("AI_AGENT", raising=False) + for name in output.AI_AGENT_ENV_MARKERS: + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv("OR_APP_NAME", raising=False) + monkeypatch.delenv("OR_SITE_URL", raising=False) + output.setup_output("default") + assert output.format_source() == "soft" + monkeypatch.setenv("AI_AGENT", "x") + output.setup_output("default") + assert output.format_source() == "hard_agent" + assert output.get_output_format() == "json" + output.setup_output("default") + monkeypatch.setitem(output._output_config, "agentic", False) + output.setup_output("default") + + # ── A SIGTERM drain ends the process by SIGTERM ──────────────────────────── @@ -239,3 +474,21 @@ def test_unreadable_stdin_is_invalid_argument(command, monkeypatch, capsys): assert result.details == {"transcript": "", "transcript_chars": 0} assert command.exit_code_for(result) == 1 assert "Couldn't read audio from stdin" in capsys.readouterr().err + + +def test_live_result_into_a_closed_stdout_is_quiet(command, monkeypatch): + """`dg -o json listen - | true`: the reader is gone before the final + Result; dg points stdout at /dev/null and carries on.""" + import sys as _sys + + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + _fmt(monkeypatch, "json") + + def boom(_text): + raise BrokenPipeError(32, "Broken pipe") + + monkeypatch.setattr(_sys.stdout, "write", boom) + r = ListenResult(status="success", source="stdin", mode="live", transcript="x") + command.output_result(r, Mock(spec=Config)) + assert silenced == [True] diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_signals.py b/packages/deepctl-cmd-listen/tests/unit/test_live_signals.py new file mode 100644 index 00000000..f5dde92c --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_signals.py @@ -0,0 +1,281 @@ +"""Real exit codes for Ctrl-C on live ``dg listen``, in a subprocess. + +A loopback WebSocket server in this process stands in for the API (fake key, +no network), and a fake ``sounddevice`` on ``PYTHONPATH`` stands in for the +microphone. ``dg listen -`` stops with exit 2 on Ctrl-C; ``dg listen --mic`` +ends with exit 0, including a Ctrl-C that lands before the session's own +handler is in place. +""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals") + +ENTRY = "import sys; from deepctl.main import main; sys.exit(main())" +FAKE_KEY = "fa1ce0000000000000000000000000000000000d" +# Generous bound for a cold interpreter start on a loaded runner; a normal +# run of each test takes about a second. +STARTUP_S = 20.0 + +FAKE_SOUNDDEVICE = ''' +import threading, time + +class RawInputStream: + """Calls the callback with 50 ms of silence until stopped.""" + + def __init__(self, samplerate, channels, dtype, callback, blocksize, **kw): + self._cb = callback + self._n = blocksize * channels + self._stop = threading.Event() + + def start(self): + def run(): + while not self._stop.is_set(): + self._cb(b"\\x10\\x00" * self._n, self._n, None, None) + time.sleep(0.05) + + threading.Thread(target=run, daemon=True).start() + + def stop(self): + self._stop.set() + + def close(self): + pass +''' + + +def _final(text: str) -> str: + return json.dumps( + { + "type": "Results", + "is_final": True, + "start": 0.0, + "duration": 1.0, + "channel": {"alternatives": [{"transcript": text, "words": []}]}, + } + ) + + +class _FakeServer: + """Sends one final after the first audio frame, and one more on + CloseStream, then closes. ``audio`` is set once audio has arrived.""" + + def __init__(self) -> None: + from websockets.sync.server import serve + + self.audio = threading.Event() + self.closestream = threading.Event() + self._server = serve(self._handle, "127.0.0.1", 0, process_request=self._http) + self.port = self._server.socket.getsockname()[1] + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @staticmethod + def _http(connection, request): + """Plain HTTP (the auth pre-flight): one fake project, as JSON.""" + if request.headers.get("Upgrade", "").lower() == "websocket": + return None + from websockets.datastructures import Headers + from websockets.http11 import Response + + body = json.dumps( + {"projects": [{"project_id": "00000000-0000-0000-0000-000000000000"}]} + ).encode() + headers = Headers( + [ + ("Content-Type", "application/json"), + ("Content-Length", str(len(body))), + ("Connection", "close"), + ] + ) + return Response(200, "OK", headers, body) + + def _handle(self, ws) -> None: + sent = False + try: + for message in ws: + if isinstance(message, bytes): + if not sent: + ws.send(_final("hello")) + sent = True + self.audio.set() + elif "CloseStream" in message: + self.closestream.set() + ws.send(_final("bye")) + ws.close() + return + except Exception: + pass + + def shutdown(self) -> None: + self._server.shutdown() + + +@pytest.fixture +def server(): + srv = _FakeServer() + yield srv + srv.shutdown() + + +@pytest.fixture +def env(tmp_path, server): + fakes = tmp_path / "fakes" + fakes.mkdir() + (fakes / "sounddevice.py").write_text(FAKE_SOUNDDEVICE) + home = tmp_path / "home" + home.mkdir() + e = { + k: v + for k, v in os.environ.items() + if not k.startswith(("DEEPGRAM_", "DEEPCTL_", "DG_")) + and k + not in ( + "AI_AGENT", + "AGENT", + "CI", + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CODEX_SANDBOX", + "CODEX_SANDBOX_NETWORK_DISABLED", + "CURSOR_AGENT", + "GEMINI_CLI", + "GOOSE_TERMINAL", + "OR_APP_NAME", + "OR_SITE_URL", + ) + } + e.update( + HOME=str(home), + XDG_CONFIG_HOME=str(home / ".config"), + TERM="xterm", + DEEPGRAM_API_KEY=FAKE_KEY, + DEEPGRAM_BASE_URL=f"http://127.0.0.1:{server.port}", + DEEPCTL_TELEMETRY_DISABLED="1", + DEEPCTL_TELEMETRY_DSN="http://k@127.0.0.1:9/1", + CI="true", # no background update check (it can hold exit for 2 s) + PYTHONPATH=os.pathsep.join( + [str(fakes), *filter(None, [os.environ.get("PYTHONPATH")])] + ), + ) + return e + + +def _start(args, env, *, stdin=subprocess.DEVNULL): + return subprocess.Popen( + [sys.executable, "-c", ENTRY, "-o", "json", "listen", *args], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + + +def _wait_for_stderr(proc, text: str, timeout: float = STARTUP_S) -> None: + """Read stderr until ``text`` appears (the banner), byte by byte.""" + assert proc.stderr is not None + seen = b"" + deadline = time.monotonic() + timeout + while text.encode() not in seen: + if time.monotonic() > deadline or proc.poll() is not None: + raise AssertionError(f"never saw {text!r}: {seen.decode(errors='replace')}") + b = proc.stderr.read(1) + if not b: + continue + seen += b + + +def _why(proc) -> str: + proc.kill() + _out, err = proc.communicate(timeout=STARTUP_S) + return f"no audio reached the server; stderr: {err.decode(errors='replace')}" + + +def _finish(proc): + # A stop drains for at most 1 s; this only bounds a hang. + out, err = proc.communicate(timeout=STARTUP_S) + return proc.returncode, out.decode(), err.decode() + + +def test_stdin_ctrl_c_exits_2(env, server): + proc = _start( + ["-", "--encoding", "linear16", "--sample-rate", "16000"], + env, + stdin=subprocess.PIPE, + ) + assert proc.stdin is not None + try: + proc.stdin.write(b"\x10\x00" * 3200) + proc.stdin.flush() + assert server.audio.wait(STARTUP_S), _why(proc) + proc.send_signal(signal.SIGINT) + rc, out, err = _finish(proc) + finally: + proc.kill() + assert rc == 2, err + assert server.closestream.is_set() + payload = json.loads(out) + assert payload["status"] == "cancelled" + assert "hello" in payload["transcript"] + assert "Traceback" not in err + + +def test_mic_ctrl_c_exits_0(env, server): + proc = _start(["--mic"], env) + try: + assert server.audio.wait(STARTUP_S), _why(proc) + proc.send_signal(signal.SIGINT) + rc, out, err = _finish(proc) + finally: + proc.kill() + assert rc == 0, err + assert server.closestream.is_set() + payload = json.loads(out) + assert payload["status"] == "success" + assert "hello" in payload["transcript"] + assert "Traceback" not in err + + +def test_mic_ctrl_c_right_after_start_exits_0(env, server): + """Ctrl-C as soon as the banner shows, before the session's handler is + installed (imports and connect still running), is still a mic stop. + --verbose keeps the status line that -o json drops in this mode.""" + proc = subprocess.Popen( + [sys.executable, "-c", ENTRY, "--verbose", "-o", "json", "listen", "--mic"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + try: + _wait_for_stderr(proc, "Listening on microphone") + proc.send_signal(signal.SIGINT) + rc, out, err = _finish(proc) + finally: + proc.kill() + assert rc == 0, err + payload = json.loads(out) + assert payload["status"] == "success" + assert "Traceback" not in err + + +def test_fake_sounddevice_is_on_path(env): + """Guard against the real PortAudio module sneaking in.""" + out = subprocess.run( + [sys.executable, "-c", "import sounddevice; print(sounddevice.__file__)"], + env=env, + capture_output=True, + text=True, + check=True, + ) + assert Path(out.stdout.strip()).parent.name == "fakes" diff --git a/packages/deepctl-core/src/deepctl_core/output.py b/packages/deepctl-core/src/deepctl_core/output.py index b7270631..c9269bc8 100644 --- a/packages/deepctl-core/src/deepctl_core/output.py +++ b/packages/deepctl-core/src/deepctl_core/output.py @@ -163,9 +163,19 @@ def __repr__(self) -> str: "verbose": False, "color": not _agentic, "agentic": _agentic, + "format_source": "default", } +def _format_source(requested: str) -> str: + """Why the output format is what it is; see format_source().""" + if requested != "default": + return "explicit" + if not _output_config["agentic"]: + return "default" + return "hard_agent" if is_ai_agent() else "soft" + + def setup_output( format_type: str = "json", quiet: bool = False, verbose: bool = False ) -> None: @@ -175,6 +185,7 @@ def setup_output( requested (format_type == "default"), auto-switch to JSON for machine-friendly output. """ + _output_config["format_source"] = _format_source(format_type) # Auto-default to JSON in agentic mode when user didn't pick a format if format_type == "default" and _output_config["agentic"]: format_type = "json" @@ -199,6 +210,7 @@ def update_output( verbose state the root group already applied. """ if format_type is not None: + _output_config["format_source"] = _format_source(format_type) if format_type == "default" and _output_config["agentic"]: format_type = "json" _output_config["format"] = format_type @@ -678,6 +690,17 @@ def get_output_format() -> str: return str(_output_config["format"]) +def format_source() -> str: + """Why the output format is what it is. + + ``"explicit"``: asked for with ``-o``. ``"hard_agent"``: JSON chosen + implicitly while an AI-agent marker (``is_ai_agent()``) is set. + ``"soft"``: JSON chosen implicitly by the TTY/CI heuristics alone. + ``"default"``: no format was chosen. + """ + return str(_output_config.get("format_source", "default")) + + def get_console() -> Console: """Get the global console instance. From 531f595aace995c857d11777b6fec9caa4c17de0 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:32:04 -0400 Subject: [PATCH 08/18] perf(speak): stream audio to pipes and players as it arrives Speak v1 wrote each chunk to stdout without flushing, so a pipe saw nothing until 8 KB had piled up (about 640 ms with mp3-sized chunks), and --play buffered the whole utterance before starting the player. Each chunk is now written and flushed as the SDK yields it, and --play with a stdin player (ffplay, paplay, aplay) starts the player before the request and feeds it chunk by chunk, as Flux already did. afplay keeps the buffered temp-file path. When the reader of stdout or the player goes away (EPIPE), the stream stops, the response is closed and the command ends quietly with exit 0; fd 1 is pointed at the null device so the exit flush can't fail. Any other stdout write error (ENOSPC, EIO, EFBIG, EDQUOT) is the new coded error output_unwritable: exit 1, two stderr lines, no envelope on stdout and no "Exception ignored". The same holds when the result document itself can't be written to stdout: EPIPE exits 0 quietly, any other error is output_unwritable. Closing a stream now waits at most 2 s. A WebSocket closing handshake the server doesn't answer, as on Ctrl-C mid-stream, used to hold the exit for up to 10 s; the close is abandoned after 2 s instead. --- .../src/deepctl_cmd_speak/command.py | 424 ++++++++++++++---- .../src/deepctl_cmd_speak/messages.py | 24 + .../src/deepctl_cmd_speak/models.py | 21 +- .../tests/unit/test_speak_command.py | 12 +- .../tests/unit/test_speak_imports.py | 59 +++ .../tests/unit/test_speak_streaming.py | 406 +++++++++++++++++ 6 files changed, 859 insertions(+), 87 deletions(-) create mode 100644 packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py create mode 100644 packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py create mode 100644 packages/deepctl-cmd-speak/tests/unit/test_speak_streaming.py diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index 3602cb62..2ca0abf3 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -3,14 +3,16 @@ from __future__ import annotations import io +import os import shutil import struct import subprocess import sys import tempfile +import threading import time import wave -from contextlib import contextmanager, suppress +from contextlib import contextmanager, nullcontext, suppress from pathlib import Path from typing import IO, TYPE_CHECKING, Any @@ -24,12 +26,19 @@ get_output_format, get_status_console, ) -from deepctl_core.output import print_warning +from deepctl_core.output import ( + _agentic, + is_ai_agent, + print_error, + print_hint, + print_warning, +) from rich.console import Console from rich.markup import escape from rich.table import Table -from .models import SpeakResult, SpeakVoicesResult, VoiceInfo +from . import messages as msg +from .models import SpeakCodedError, SpeakResult, SpeakVoicesResult, VoiceInfo if TYPE_CHECKING: from collections.abc import Iterator @@ -226,6 +235,107 @@ def _play_audio(player: str, audio: bytes, *, suffix: str) -> None: sink.write(audio) +def _prog() -> str: + """The program name to show in messages and ``next_step``. + + The bare name when it resolves on PATH to the running executable; + otherwise ``dg``. + """ + try: + argv0 = sys.argv[0] + name = os.path.basename(argv0) + if name in ("dg", "deepgram", "deepctl"): + found = shutil.which(name) + if found and os.path.realpath(found) == os.path.realpath(argv0): + return name + except Exception: + pass + return "dg" + + +def _silence_stdout() -> None: + """Point fd 1 at the null device, so buffered output and later writes, + including the flush at exit, succeed quietly. Used once the reader of + stdout went away (EPIPE) or stdout can't be written (ENOSPC, EIO).""" + try: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 1) + os.close(devnull) + except OSError: + pass + + +def _os_error_text(exc: OSError) -> str: + """The OS error text for a message: printable, one line, bounded.""" + text = exc.strerror or str(exc) or exc.__class__.__name__ + return "".join(ch if ch.isprintable() else " " for ch in text).strip()[:200] + + +#: Longest wait for an audio stream to close: a WebSocket closing handshake +#: the server doesn't answer (Ctrl-C mid-stream) must not hold up the exit. +_CLOSE_TIMEOUT_S = 2.0 + + +def _close(*streams: object, timeout: float = _CLOSE_TIMEOUT_S) -> None: + """Close audio iterators (and the responses behind them), quietly and in + order, and wait at most ``timeout`` seconds for all of them together.""" + closes = [c for c in (getattr(s, "close", None) for s in streams) if c] + if not closes: + return + + def run() -> None: + for close in closes: + with suppress(BaseException): + close() + + closer = threading.Thread(target=run, name="deepctl-speak-close", daemon=True) + closer.start() + closer.join(timeout) + + +def _coded( + text: str, + *, + code: str, + hint_human: str, + hint_agent: str, + next_step: str | None = None, + details: dict[str, Any] | None = None, +) -> SpeakCodedError: + """Report a coded failure on stderr (two lines) and return its envelope.""" + hint = hint_agent if (_agentic or is_ai_agent()) else hint_human + print_error(escape(text)) + print_hint(escape(hint)) + return SpeakCodedError( + error=text, code=code, hint=hint, next_step=next_step, details=details + ) + + +def _output_unwritable(exc: OSError, *, path: str) -> SpeakCodedError: + """``output_unwritable`` for a failed write to ``path`` (``-`` = stdout).""" + reason = _os_error_text(exc) + return _coded( + msg.E_OUTPUT_UNWRITABLE.format( + output="stdout" if path == "-" else path, reason=reason + ), + code=msg.CODE_OUTPUT_UNWRITABLE, + hint_human=msg.H_OUTPUT_UNWRITABLE, + hint_agent=msg.A_OUTPUT_UNWRITABLE.format(prog=_prog(), reason=reason), + details={"path": path, "reason": reason}, + ) + + +def _stdout_unwritable(exc: OSError) -> SpeakCodedError: + """stdout failed (not EPIPE): silence fd 1 first, so the envelope goes + to stderr only and the flush at exit can't fail again.""" + _silence_stdout() + return _output_unwritable(exc, path="-") + + +class _NoAudio(Exception): + """The stream ended without a single audio byte.""" + + def _fail(message: str) -> BaseResult: """An error result whose message the user actually sees. @@ -512,7 +622,7 @@ def handle( auth_manager: AuthManager, client: DeepgramClient, **kwargs: Any, - ) -> BaseResult | None: + ) -> BaseResult | SpeakCodedError | None: text = kwargs.get("text") output_path = kwargs.get("output") model = kwargs.get("model") or _DEFAULT_MODEL @@ -675,9 +785,12 @@ def handle( except BrokenPipeError: # The player exited first (ffplay's "q", for # instance). That is the user stopping - # playback, not a failure — but keep draining - # the stream so a -o file is still complete. + # playback, not a failure. With -o, keep + # draining so the file is still complete; + # otherwise stop and close the stream. player_gone = True + if not output_path: + break if output_path and pcm: # Save before waiting on the player (that wait @@ -694,6 +807,8 @@ def handle( raise except Exception as e: raise click.ClickException(f"Flux streaming failed: {e}") + finally: + _close(stream) if prog.total == 0: raise click.ClickException( @@ -786,26 +901,22 @@ def handle( # "streaming by default"). For linear16 we emit a streaming WAV # header before the first frame so `| ffplay -` plays with no # extra flags; raw mulaw/alaw stream as-is. - out = sys.stdout.buffer - wrote_header = False + header = ( + _streaming_wav_header(sample_rate=int(eff_sample_rate)) + if eff_encoding == "linear16" + else b"" + ) try: - for chunk in stream: - if not chunk: - continue - # stdout now carries audio: a later failure is reported - # on stderr only, never as JSON after the audio. - self.stdout_is_protocol = True - if eff_encoding == "linear16" and not wrote_header: - out.write( - _streaming_wav_header(sample_rate=int(eff_sample_rate)) - ) - wrote_header = True - out.write(chunk) - out.flush() + _, outcome = self._stream_to_stdout(stream, header=header) except click.ClickException: raise except Exception as e: raise click.ClickException(f"Flux streaming failed: {e}") + finally: + _close(stream) + if outcome is not True: + # The reader went away (quiet, exit 0) or stdout failed. + return outcome or None if prog.total == 0: # Guard before anything is written: the header is only emitted @@ -826,9 +937,10 @@ def handle( return None # REST path (Speak v1, including Aura and unknown-model pass-through). + # Each chunk goes on as the SDK yields it: to stdout (flushed), or to + # a stdin-reading player, so neither waits for the whole utterance. + console.print(f"[blue]Generating speech with {model}...[/blue]") try: - console.print(f"[blue]Generating speech with {model}...[/blue]") - audio_iter = client.speak_text( text=text, model=model, @@ -836,51 +948,26 @@ def handle( container=container, sample_rate=sample_rate, ) + except Exception as e: + raise click.ClickException(f"Error generating speech: {e}") - total_bytes = 0 - - if player: - # Playback needs the full utterance, so buffer it. When -o is - # also given, save the same bytes to the file too. - audio = bytearray() - for chunk in audio_iter: - audio.extend(chunk) - audio_bytes = bytes(audio) - total_bytes = len(audio_bytes) - - if not audio_bytes: - return _fail("TTS returned no audio.") - - if output_path: - Path(output_path).write_bytes(audio_bytes) - console.print( - f"[green]Audio saved to {output_path}[/green] " - f"({total_bytes:,} bytes)" - ) - - suffix = _play_suffix( - is_flux=False, encoding=encoding, container=container - ) - console.print(f"[blue]Playing audio ({player})...[/blue]") - _play_audio(player, audio_bytes, suffix=suffix) - - message = ( - f"Audio saved to {output_path}" - if output_path - else f"Played {total_bytes:,} bytes" + try: + if player in _STDIN_PLAYER_ARGV: + return self._v1_play_streaming( + audio_iter, player=player, output_path=output_path, model=model ) - return SpeakResult( - status="success", - message=message, - output_path=output_path or "", + if player: + return self._v1_play_buffered( + audio_iter, + player=player, + output_path=output_path, model=model, - bytes_written=total_bytes, - played=True, + encoding=encoding, + container=container, ) - elif output_path: - # Write to file - out = Path(output_path) - with open(out, "wb") as f: + if output_path: + total_bytes = 0 + with open(Path(output_path), "wb") as f: for chunk in audio_iter: f.write(chunk) total_bytes += len(chunk) @@ -888,7 +975,6 @@ def handle( console.print( f"[green]Audio saved to {output_path}[/green] ({total_bytes:,} bytes)" ) - return SpeakResult( status="success", message=f"Audio saved to {output_path}", @@ -896,23 +982,17 @@ def handle( model=model, bytes_written=total_bytes, ) - else: - # Write to stdout (piping) - stdout_buffer = sys.stdout.buffer - for chunk in audio_iter: - # stdout now carries audio: a later failure is reported - # on stderr only, never as JSON after the audio. - self.stdout_is_protocol = True - stdout_buffer.write(chunk) - total_bytes += len(chunk) - stdout_buffer.flush() - - console.print(f"[green]✓ Wrote {total_bytes:,} bytes to stdout[/green]") - # Return None so the framework skips output_result — otherwise - # agentic/CI (or --output json) mode serializes JSON onto the - # same stdout as the audio, corrupting a `> out` redirect. The - # summary above went to the stderr console. - return None + + total_bytes, outcome = self._stream_to_stdout(audio_iter) + if outcome is not True: + # The reader went away (quiet, exit 0) or stdout failed. + return outcome or None + console.print(f"[green]✓ Wrote {total_bytes:,} bytes to stdout[/green]") + # Return None so the framework skips output_result — otherwise + # agentic/CI (or --output json) mode serializes JSON onto the + # same stdout as the audio, corrupting a `> out` redirect. The + # summary above went to the stderr console. + return None except click.ClickException: raise @@ -922,6 +1002,186 @@ def handle( # still exits 0. Reachable now that unknown models (e.g. a bare # "flux" typo) route here instead of the raising v2 path. raise click.ClickException(f"Error generating speech: {e}") + finally: + _close(audio_iter) + + def _stream_to_stdout( + self, stream: Iterator[bytes], *, header: bytes = b"" + ) -> tuple[int, SpeakCodedError | bool]: + """Write each chunk to stdout and flush it as it arrives. + + ``header`` goes out just before the first chunk, so a stream with no + audio writes nothing. Returns the audio bytes written and ``True`` + when the stream ran to its end; ``False`` when the reader of stdout + went away (EPIPE: fd 1 now points at the null device and the caller + ends quietly); or the ``output_unwritable`` envelope when the write + failed (ENOSPC, EIO, ...). Errors from the stream itself propagate. + """ + out = sys.stdout.buffer + total = 0 + for chunk in stream: + if not chunk: + continue + # stdout now carries audio: a later failure is reported on + # stderr only, never as JSON after the audio. + self.stdout_is_protocol = True + try: + if header: + out.write(header) + header = b"" + out.write(chunk) + out.flush() + except BrokenPipeError: + _silence_stdout() + return total, False + except OSError as exc: + return total, _stdout_unwritable(exc) + total += len(chunk) + return total, True + + def _v1_play_streaming( + self, + audio_iter: Iterator[bytes], + *, + player: str, + output_path: str | None, + model: str, + ) -> BaseResult: + """Speak v1 into a stdin-reading player, chunk by chunk. + + The player is started before the request, so sound starts as the + first chunk arrives. With -o, every chunk also goes to the file. + """ + console.print(f"[blue]Playing audio ({player})...[/blue]") + total = 0 + player_gone = False + try: + with ( + _player_stdin(player) as sink, + open(Path(output_path), "wb") + if output_path + else nullcontext() as saved, + ): + for chunk in audio_iter: + if not chunk: + continue + total += len(chunk) + if saved is not None: + saved.write(chunk) + if player_gone: + continue + try: + sink.write(chunk) + sink.flush() + except BrokenPipeError: + # The user stopped playback (ffplay's "q"). With -o, + # keep draining so the file is complete; otherwise + # stop and close the response. + player_gone = True + if saved is None: + break + if not total: + # Kill the player rather than hand it an empty stream. + raise _NoAudio + except _NoAudio: + return _fail("TTS returned no audio.") + + if output_path: + console.print( + f"[green]Audio saved to {output_path}[/green] ({total:,} bytes)" + ) + return SpeakResult( + status="success", + message=( + f"Audio saved to {output_path}" + if output_path + else f"Played {total:,} bytes" + ), + output_path=output_path or "", + model=model, + bytes_written=total, + played=True, + ) + + def _v1_play_buffered( + self, + audio_iter: Iterator[bytes], + *, + player: str, + output_path: str | None, + model: str, + encoding: str | None, + container: str | None, + ) -> BaseResult: + """Speak v1 into a player that has no stdin mode (afplay). + + It is handed a complete temp file, so the utterance is buffered. + """ + audio = bytearray() + for chunk in audio_iter: + audio.extend(chunk) + audio_bytes = bytes(audio) + total_bytes = len(audio_bytes) + + if not audio_bytes: + return _fail("TTS returned no audio.") + + if output_path: + Path(output_path).write_bytes(audio_bytes) + console.print( + f"[green]Audio saved to {output_path}[/green] ({total_bytes:,} bytes)" + ) + + suffix = _play_suffix(is_flux=False, encoding=encoding, container=container) + console.print(f"[blue]Playing audio ({player})...[/blue]") + _play_audio(player, audio_bytes, suffix=suffix) + + return SpeakResult( + status="success", + message=( + f"Audio saved to {output_path}" + if output_path + else f"Played {total_bytes:,} bytes" + ), + output_path=output_path or "", + model=model, + bytes_written=total_bytes, + played=True, + ) + + # ── Result on stdout ────────────────────────────────────────────── + + #: Writing the result to stdout failed; the run exits 1. + _output_failed: bool = False + + def output_result(self, result: Any, config: Config) -> None: + """Write the result; a stdout that can't take it is + ``output_unwritable`` (exit 1), never an exit-time "Exception + ignored".""" + try: + super().output_result(result, config) + if result is not None: + sys.stdout.flush() + except BrokenPipeError: + _silence_stdout() + except SystemExit as exc: + # rich handles EPIPE itself: it points stdout at the null device + # and raises SystemExit(1). The reader went away, which ends the + # run quietly with 0, as the streaming paths do. + if not isinstance(exc.__context__, BrokenPipeError): + raise + _silence_stdout() + except OSError as exc: + _silence_stdout() + if isinstance(result, SpeakCodedError): + return # already reported on stderr; it exits 1 anyway + _output_unwritable(exc, path="-") + self._output_failed = True + + def exit_code_for(self, result: Any) -> int: + if self._output_failed: + return 1 + return super().exit_code_for(result) def _list_voices(self, client: DeepgramClient) -> BaseResult: """List available TTS voices as a table (mirrors `dg models`).""" diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py new file mode 100644 index 00000000..87f606a2 --- /dev/null +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py @@ -0,0 +1,24 @@ +"""User-facing strings for ``dg speak``, keyed by message ID. + +Every ID and string matches the error catalogue (``error_codes.json``) +exactly; the attribute name is the ID with ``-`` as ``_``. +""" + +from __future__ import annotations + +# ── Catalogue ────────────────────────────────────────────────────────────── + +CODE_OUTPUT_UNWRITABLE = "output_unwritable" + +# output_unwritable: a write to stdout or the audio file failed. +# {reason} is details.reason: an OS error text or one of dg's own (sinks.py). +E_OUTPUT_UNWRITABLE = "Couldn't write the output to {output}: {reason}." +H_OUTPUT_UNWRITABLE = ( + "Choose another destination, or fix this one (free space, permissions), " + "then try again." +) +A_OUTPUT_UNWRITABLE = ( + "If you chose this destination, choose another one. Otherwise tell the " + "user {prog} couldn't write its output ({reason}), and don't retry until " + "it's fixed." +) diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/models.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/models.py index 45dcad25..c865da4d 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/models.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/models.py @@ -2,7 +2,9 @@ from __future__ import annotations -from deepctl_core import BaseResult +from typing import Any + +from deepctl_core import BaseResult, CodedErrorResult from pydantic import BaseModel, Field @@ -23,3 +25,20 @@ class VoiceInfo(BaseModel): class SpeakVoicesResult(BaseResult): voices: list[VoiceInfo] = Field(default_factory=list) count: int = 0 + + +class SpeakCodedError(CodedErrorResult): + """A coded error from ``dg speak``, keys in the catalogue's order. + + ``details`` is left out of the payload when there are none. + """ + + next_step: str | None = None + schema_version: int = 1 + details: dict[str, Any] | None = None + + def model_dump(self, **kwargs: Any) -> dict[str, Any]: + data = super().model_dump(**kwargs) + if data.get("details") is None: + data.pop("details", None) + return data diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py index df476f88..640d3bce 100644 --- a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py @@ -6,7 +6,7 @@ import wave from contextlib import contextmanager from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import Mock, call, patch import click import pytest @@ -905,9 +905,13 @@ def test_handle_write_to_stdout( assert result is None - mock_stdout_buffer.write.assert_any_call(b"chunk1") - mock_stdout_buffer.write.assert_any_call(b"chunk2") - mock_stdout_buffer.flush.assert_called_once() + # Each chunk is flushed as it arrives, not once at the end. + assert mock_stdout_buffer.mock_calls == [ + call.write(b"chunk1"), + call.flush(), + call.write(b"chunk2"), + call.flush(), + ] # ── --list-voices ──────────────────────────────────────────────── diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py new file mode 100644 index 00000000..9d5087ba --- /dev/null +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py @@ -0,0 +1,59 @@ +"""The speak modules import no network stack at module level. + +The speak package's copy of ``tests/unit/test_command_imports.py``, which +runs the same check for every built-in command. ``dg speak --help`` imports these modules, so +a module-level ``asyncio``, ``ssl``, ``httpx``, ``websockets`` or SDK +import would be paid by every help and every run. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +FORBIDDEN = ("asyncio", "ssl", "httpx", "websockets", "deepgram") + +CHECK = ( + "import importlib, sys; importlib.import_module(sys.argv[1]); " + "bad = sorted({m for m in sys.modules for f in sys.argv[2:] " + "if m == f or m.startswith(f + '.')}); " + "print(' '.join(bad)); sys.exit(1 if bad else 0)" +) + + +def _env(tmp_path: Path) -> dict[str, str]: + env = { + k: v for k, v in os.environ.items() if not k.startswith(("DEEPGRAM_", "PYTHON")) + } + env.update( + HOME=str(tmp_path), + DEEPCTL_TELEMETRY_DISABLED="1", + DEEPCTL_TELEMETRY_DSN="http://k@127.0.0.1:9/1", + ) + return env + + +@pytest.mark.parametrize( + "module", + [ + "deepctl_cmd_speak.command", + "deepctl_cmd_speak.messages", + "deepctl_cmd_speak.models", + ], +) +def test_speak_module_imports_no_network_stack(module: str, tmp_path: Path) -> None: + proc = subprocess.run( + [sys.executable, "-c", CHECK, module, *FORBIDDEN], + capture_output=True, + text=True, + env=_env(tmp_path), + timeout=120, + ) + assert proc.returncode == 0, f"{module} imports: {proc.stdout}{proc.stderr[-1500:]}" diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_streaming.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_streaming.py new file mode 100644 index 00000000..e122c93e --- /dev/null +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_streaming.py @@ -0,0 +1,406 @@ +"""Streaming audio to stdout and players as it arrives. + +Also the stdout failure modes: EPIPE ends quietly, any other write error +is ``output_unwritable`` with fd 1 silenced. +""" + +from __future__ import annotations + +import errno +from contextlib import contextmanager +from typing import Any +from unittest.mock import Mock, patch + +import pytest +from deepctl_cmd_speak import command as speak +from deepctl_cmd_speak.command import SpeakCommand +from deepctl_cmd_speak.models import SpeakCodedError, SpeakResult +from deepctl_core import AuthManager, Config, DeepgramClient + + +class _Recorder: + """An event log shared by a fake stream, a fake stdout and a fake player.""" + + def __init__(self) -> None: + self.events: list[tuple[str, Any]] = [] + self.closed = False + + +class _Stream: + """An SDK-like audio iterator that logs each hand-out and its close().""" + + def __init__(self, rec: _Recorder, chunks: list[bytes]) -> None: + self._rec = rec + self._chunks = list(chunks) + + def __iter__(self) -> _Stream: + return self + + def __next__(self) -> bytes: + if not self._chunks: + raise StopIteration + chunk = self._chunks.pop(0) + self._rec.events.append(("yield", chunk)) + return chunk + + def close(self) -> None: + self._rec.closed = True + + +class _Sink: + """A file-like stdout/player stdin that logs writes and flushes.""" + + def __init__(self, rec: _Recorder, fail_on: int | None = None, exc=None) -> None: + self._rec = rec + self._fail_on = fail_on + self._exc = exc + self._writes = 0 + + def write(self, data: bytes) -> int: + self._writes += 1 + if self._fail_on is not None and self._writes >= self._fail_on: + raise self._exc + self._rec.events.append(("write", data)) + return len(data) + + def flush(self) -> None: + self._rec.events.append(("flush", None)) + + +def _run(cmd: SpeakCommand, client: Mock, **kwargs: Any): + base = { + "text": "Hello", + "output": None, + "model": "aura-2-asteria-en", + "encoding": None, + "container": None, + "sample_rate": None, + "file": None, + } + base.update(kwargs) + return cmd.handle( + config=Mock(spec=Config), + auth_manager=Mock(spec=AuthManager), + client=client, + **base, + ) + + +@pytest.fixture +def fake_sys(): + with patch("deepctl_cmd_speak.command.sys") as mock_sys: + mock_sys.stdin.isatty.return_value = True + mock_sys.stdout.isatty.return_value = False + mock_sys.argv = ["dg", "speak", "Hello"] + yield mock_sys + + +@pytest.fixture +def silenced(): + with patch("deepctl_cmd_speak.command._silence_stdout") as fn: + yield fn + + +class TestV1ToStdout: + def test_each_chunk_is_written_and_flushed_before_the_next(self, fake_sys): + rec = _Recorder() + fake_sys.stdout.buffer = _Sink(rec) + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a", b"b", b"c"]) + + assert _run(SpeakCommand(), client) is None + assert rec.events == [ + ("yield", b"a"), + ("write", b"a"), + ("flush", None), + ("yield", b"b"), + ("write", b"b"), + ("flush", None), + ("yield", b"c"), + ("write", b"c"), + ("flush", None), + ] + assert rec.closed + + def test_epipe_stops_closes_the_response_and_ends_quietly( + self, fake_sys, silenced, capsys + ): + rec = _Recorder() + fake_sys.stdout.buffer = _Sink(rec, fail_on=2, exc=BrokenPipeError()) + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a", b"b", b"c", b"d"]) + cmd = SpeakCommand() + + result = _run(cmd, client) + + assert result is None + assert cmd.exit_code_for(result) == 0 + # Stopped at the failed write: "c" and "d" were never requested. + assert [e for e in rec.events if e[0] == "yield"] == [ + ("yield", b"a"), + ("yield", b"b"), + ] + assert rec.closed + silenced.assert_called_once() + assert "Error" not in capsys.readouterr().err + + @pytest.mark.parametrize("err", [errno.ENOSPC, errno.EIO, errno.EFBIG]) + def test_other_write_errors_are_output_unwritable( + self, fake_sys, silenced, capsys, err + ): + rec = _Recorder() + exc = OSError(err, __import__("os").strerror(err)) + fake_sys.stdout.buffer = _Sink(rec, fail_on=1, exc=exc) + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a", b"b"]) + cmd = SpeakCommand() + + result = _run(cmd, client) + + assert isinstance(result, SpeakCodedError) + assert cmd.exit_code_for(result) == 1 + assert result.code == "output_unwritable" + assert result.next_step is None + assert result.details == {"path": "-", "reason": exc.strerror} + assert result.error == (f"Couldn't write the output to stdout: {exc.strerror}.") + assert rec.closed + # fd 1 is pointed at the null device before anything else is said, + # so the envelope can't land on stdout and the exit flush can't fail. + silenced.assert_called_once() + err_lines = [ + line for line in capsys.readouterr().err.splitlines() if line.strip() + ] + assert any(result.error in line for line in err_lines) + assert any(result.hint in line for line in err_lines) + assert cmd.stdout_is_protocol is True + + def test_error_text_is_sanitized(self, fake_sys, silenced): + rec = _Recorder() + exc = OSError(errno.EIO, "bad\x1b[31m\nthing" + "x" * 300) + fake_sys.stdout.buffer = _Sink(rec, fail_on=1, exc=exc) + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a"]) + + result = _run(SpeakCommand(), client) + + reason = result.details["reason"] + assert "\x1b" not in reason and "\n" not in reason + assert len(reason) <= 200 + + +class TestFluxToStdout: + def test_header_then_each_frame_flushed(self, fake_sys): + rec = _Recorder() + fake_sys.stdout.buffer = _Sink(rec) + client = Mock(spec=DeepgramClient) + client.speak_text_stream.return_value = _Stream(rec, [b"\x01\x00", b"\x02\x00"]) + + assert _run(SpeakCommand(), client, model="flux-alexis-en") is None + writes = [e for e in rec.events if e[0] != "yield"] + assert writes[0][0] == "write" and writes[0][1][:4] == b"RIFF" + assert writes[1:] == [ + ("write", b"\x01\x00"), + ("flush", None), + ("write", b"\x02\x00"), + ("flush", None), + ] + + def test_enospc_is_output_unwritable(self, fake_sys, silenced): + rec = _Recorder() + exc = OSError(errno.ENOSPC, "No space left on device") + fake_sys.stdout.buffer = _Sink(rec, fail_on=1, exc=exc) + client = Mock(spec=DeepgramClient) + client.speak_text_stream.return_value = _Stream(rec, [b"\x01\x00"]) + cmd = SpeakCommand() + + result = _run(cmd, client, model="flux-alexis-en") + + assert isinstance(result, SpeakCodedError) + assert result.code == "output_unwritable" + assert cmd.exit_code_for(result) == 1 + silenced.assert_called_once() + + def test_epipe_ends_quietly(self, fake_sys, silenced): + rec = _Recorder() + fake_sys.stdout.buffer = _Sink(rec, fail_on=1, exc=BrokenPipeError()) + client = Mock(spec=DeepgramClient) + client.speak_text_stream.return_value = _Stream(rec, [b"\x01\x00", b"\x02"]) + cmd = SpeakCommand() + + result = _run(cmd, client, model="flux-alexis-en") + + assert result is None + assert cmd.exit_code_for(result) == 0 + silenced.assert_called_once() + + +class TestV1Play: + @pytest.fixture + def player(self, monkeypatch): + """A stdin player that logs its start and every write.""" + rec = _Recorder() + state: dict[str, Any] = {"fail_on": None} + + @contextmanager + def fake_player_stdin(name): + rec.events.append(("spawn", name)) + yield _Sink(rec, fail_on=state["fail_on"], exc=BrokenPipeError()) + rec.events.append(("waited", name)) + + monkeypatch.setattr(speak, "_player_stdin", fake_player_stdin) + monkeypatch.setattr(speak, "_find_audio_player", lambda: "ffplay") + return rec, state + + def test_player_starts_first_and_gets_each_chunk_as_it_arrives( + self, fake_sys, player + ): + rec, _ = player + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a", b"b"]) + + result = _run(SpeakCommand(), client, play=True, encoding="linear16") + + assert isinstance(result, SpeakResult) + assert result.played is True + assert result.bytes_written == 2 + assert rec.events == [ + ("spawn", "ffplay"), + ("yield", b"a"), + ("write", b"a"), + ("flush", None), + ("yield", b"b"), + ("write", b"b"), + ("flush", None), + ("waited", "ffplay"), + ] + assert rec.closed + + def test_player_quit_stops_and_closes_the_response(self, fake_sys, player): + rec, state = player + state["fail_on"] = 1 + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, [b"a", b"b", b"c"]) + + result = _run(SpeakCommand(), client, play=True, encoding="linear16") + + assert isinstance(result, SpeakResult) + assert [e for e in rec.events if e[0] == "yield"] == [("yield", b"a")] + assert rec.closed + + def test_no_audio_fails(self, fake_sys, player): + rec, _ = player + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = _Stream(rec, []) + cmd = SpeakCommand() + + result = _run(cmd, client, play=True, encoding="linear16") + + assert result.status == "error" + assert "no audio" in result.message + + +class TestResultOnStdout: + """The SpeakResult itself can't be written to stdout.""" + + def _result(self): + return SpeakResult(status="success", output_path="a.wav", bytes_written=3) + + def test_full_stdout_is_output_unwritable_and_exits_1(self, silenced, capsys): + cmd = SpeakCommand() + with patch.object( + speak.BaseCommand, + "output_result", + side_effect=OSError(errno.ENOSPC, "No space left on device"), + ): + cmd.output_result(self._result(), Mock(spec=Config)) + silenced.assert_called_once() + assert cmd.exit_code_for(self._result()) == 1 + err = capsys.readouterr().err + assert "Couldn't write the output to stdout: No space left on device." in err + + def test_epipe_on_the_result_is_quiet_and_exits_0(self): + """The real path: rich's own EPIPE handling under -o json, with the + reader of stdout already gone.""" + import subprocess + import sys as _sys + + code = ( + "import sys\n" + "from deepctl_core.output import setup_output\n" + "setup_output('json')\n" + "from deepctl_cmd_speak.command import SpeakCommand\n" + "from deepctl_cmd_speak.models import SpeakResult\n" + "cmd = SpeakCommand()\n" + "r = SpeakResult(status='success', output_path='a.wav', message='x' * 70000)\n" + "cmd.output_result(r, None)\n" + "sys.stderr.write(f'RC={cmd.exit_code_for(r)}\\n')\n" + ) + p = subprocess.Popen( + [_sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={**__import__("os").environ, "DEEPCTL_TELEMETRY_DISABLED": "1"}, + ) + p.stdout.close() # the reader is gone before the result is written + err = p.stderr.read().decode() + assert p.wait(60) == 0, err + assert "RC=0" in err + assert "Traceback" not in err and "Exception ignored" not in err + + def test_a_coded_error_is_not_reported_twice(self, silenced, capsys): + cmd = SpeakCommand() + coded = SpeakCodedError(error="x", code="output_unwritable", hint="h") + with patch.object( + speak.BaseCommand, "output_result", side_effect=OSError(errno.EIO, "I/O") + ): + cmd.output_result(coded, Mock(spec=Config)) + silenced.assert_called_once() + assert capsys.readouterr().err == "" + + +class TestCloseIsBounded: + def test_a_hanging_close_is_abandoned_after_the_timeout(self): + import threading + import time + + release = threading.Event() + + class Hangs: + def close(self): + release.wait(5) + + t0 = time.monotonic() + speak._close(Hangs(), timeout=0.2) + assert time.monotonic() - t0 < 3.0 # the close itself hangs 5 s + release.set() + + def test_several_closes_share_one_timeout(self): + """Closing the stream and the response behind it waits once, not + once per close.""" + import threading + import time + + release = threading.Event() + closed: list[str] = [] + + class Hangs: + def __init__(self, name): + self.name = name + + def close(self): + closed.append(self.name) + release.wait(10) + + # One 2 s wait; one per close would be 4 s. + t0 = time.monotonic() + speak._close(Hangs("a"), Hangs("b"), timeout=2.0) + assert time.monotonic() - t0 < 3.5 + assert closed == ["a"] # in order; b waits behind the stalled a + release.set() + + def test_close_errors_are_quiet(self): + class Fails: + def close(self): + raise RuntimeError("boom") + + speak._close(Fails()) From cd70665a7a009b75aded7c54aa459ed4c91f9b10 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:43:40 -0400 Subject: [PATCH 09/18] feat(speak): add --out-file and deprecate -o/--output for the audio file --out-file FILE writes the audio to a file without ever replacing one by accident. Before the request, the target is checked with lstat: an existing file or symlink is refused (E-OUT-FILE-EXISTS) unless --force is given, and a directory, a block device, a socket, stderr itself, a --force over a /dev/fd entry, or a missing or unwritable parent is output_unwritable, so nothing is sent or billed. A character device or FIFO is written as it is, with no --force. It is opened before the request without blocking, so a FIFO with no reader is refused rather than waited on, and the opened file must be the one that was checked: a path swapped for a regular file in the meantime is never written. A new file is opened when the first chunk arrives, with O_EXCL as the race guard, so a failed request, a cut stream or an empty one leaves no file; a partial file is removed on any later error, and a failed write is output_unwritable with the envelope on stdout. --force writes a 0600 temp file in the same directory and replaces the target at the end, keeping an existing file's permission bits (never setuid, setgid or sticky), so a symlink is replaced rather than written through. Flux still buffers its WAV so the header carries the exact length. During the request and stream, SIGTERM and SIGHUP remove the partial or temp file (an old file is kept) and then end the process by the signal; Ctrl-C does the same cleanup and cancels as before. If the unwind stalls, as in a WebSocket close the server doesn't answer, the signal is delivered again after 4 s and ends the run. A signal that was ignored when dg started (nohup, a background job, trap '') stays ignored and the file is completed. Signals that arrive while the file is created, committed or removed are held until that step is done; the most severe of them is then delivered. On Windows, dg never calls os.kill on its own process. When --out-file is stdout itself (/dev/stdout), the audio is the only thing written there (fd 1 is checked again, not reopened) and the summary goes to stderr; an explicit -o json or -o yaml is refused before the request (E-STDOUT-BYTES). -o/--output FILE keeps working with one DEPRECATED line per process, printed even under -q, and now replaces the file through a temp file, so a failed run leaves the old file intact. A bare format value (-o json) is refused with a runnable next_step, and -o together with --out-file is refused. Help, examples and the legacy hints now name --out-file. Changes to -o FILE for the changelog: a symlink at FILE is replaced, not written through; the directory holding FILE must be writable; and FILE becomes a new file with the same permission bits (setuid, setgid and sticky are dropped), so hard links to it keep the old bytes. --- .../src/deepctl_cmd_speak/command.py | 1216 +++++++++++----- .../src/deepctl_cmd_speak/messages.py | 55 + .../src/deepctl_cmd_speak/sinks.py | 538 ++++++++ .../tests/unit/test_speak_command.py | 24 +- .../tests/unit/test_speak_imports.py | 1 + .../tests/unit/test_speak_output.py | 1227 +++++++++++++++++ .../src/deepctl_core/base_group_command.py | 1 + .../src/deepctl_core/plugin_manager.py | 2 + .../deepctl-core/src/deepctl_core/signals.py | 207 +++ .../tests/unit/test_base_group.py | 17 + .../tests/unit/test_plugin_manager.py | 22 + .../deepctl-core/tests/unit/test_signals.py | 255 ++++ src/deepctl/_manifest.d/speak.json | 48 +- 13 files changed, 3298 insertions(+), 315 deletions(-) create mode 100644 packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py create mode 100644 packages/deepctl-cmd-speak/tests/unit/test_speak_output.py create mode 100644 packages/deepctl-core/src/deepctl_core/signals.py create mode 100644 packages/deepctl-core/tests/unit/test_signals.py diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index 2ca0abf3..29005ff2 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -1,10 +1,18 @@ -"""Speak (text-to-speech) command for deepctl.""" +"""Speak (text-to-speech) command for deepctl. + +Signal handling (SIGINT/SIGTERM/SIGHUP during a request) is written for +the dg process; running this command in-process inside another program +isn't supported for signal semantics. +""" from __future__ import annotations import io import os +import secrets +import shlex import shutil +import signal import struct import subprocess import sys @@ -12,9 +20,12 @@ import threading import time import wave -from contextlib import contextmanager, nullcontext, suppress +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from typing import IO, TYPE_CHECKING, Any +from urllib.parse import urlsplit import click from deepctl_core import ( @@ -28,20 +39,27 @@ ) from deepctl_core.output import ( _agentic, + format_source, is_ai_agent, print_error, print_hint, print_warning, + stderr_console, ) +from deepctl_core.signals import coalescing as _coalescing +from deepctl_core.signals import defer_in_importlib as _defer_in_importlib +from deepctl_core.signals import die_by_signal as _die_by_signal +from deepctl_core.signals import send_to_main as _send_to_main from rich.console import Console from rich.markup import escape from rich.table import Table from . import messages as msg +from . import sinks from .models import SpeakCodedError, SpeakResult, SpeakVoicesResult, VoiceInfo if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Callable, Iterator console = get_status_console() # Tables and other stdout-bound rendering (only used when no audio goes to @@ -157,9 +175,9 @@ def _check_playable( ) if raw: return ( - f"--play cannot play raw {eff_encoding} audio: it has no container " - "for the player to detect. Use the default linear16 audio, add " - "--container wav (Aura), or save it with -o." + f"--play can't play raw {eff_encoding} audio because the player " + "can't tell its format. Use linear16 (the default), add " + "--container wav (Aura only), or save it with --out-file FILE." ) unplayable = _UNPLAYABLE_BY_PLAYER.get(player, ()) @@ -170,9 +188,9 @@ def _check_playable( if e not in unplayable ) return ( - f"'{player}' cannot decode {eff_encoding}; it plays {playable}. " - "Pick one of those with --encoding, install ffmpeg (ffplay) to play " - f"{eff_encoding}, or save it with -o." + f"'{player}' can't decode {eff_encoding}; it plays {playable}. " + "Pick one of those with --encoding, install ffmpeg (for ffplay) to " + f"play {eff_encoding}, or save it with --out-file FILE." ) return None @@ -336,6 +354,311 @@ class _NoAudio(Exception): """The stream ended without a single audio byte.""" +# Raised by an AudioFile; they pass through the "request failed" handlers. +_SINK_ERRORS = (sinks.Exists, sinks.WriteFailed, sinks.ReaderGone) + +# Values of `speak -o` that are output formats, not file names. +_FORMAT_WORDS = ("json", "yaml", "text", "table", "csv") + +# Root -o values: the formats a corrected command line can actually use. +_ROOT_FORMATS = ("json", "yaml", "table", "csv") + +# Root options that take a value, used when no Click context is available. +_ROOT_VALUE_OPTS = frozenset( + {"--config", "-c", "--profile", "-p", "--base-url", "--api-key", "--output", "-o"} +) + +_deprecation_shown = False + + +def _argv() -> list[str]: + """The user's arguments after the program name (a seam for tests).""" + return list(sys.argv[1:]) + + +def _root_value_opts() -> frozenset[str]: + """Root options that take a value, from the running root command.""" + ctx = click.get_current_context(silent=True) + if ctx is None: + return _ROOT_VALUE_OPTS + opts: set[str] = set() + for param in ctx.find_root().command.params: + if isinstance(param, click.Option) and (param.is_flag or param.count): + continue + opts.update(param.opts) + opts.update(param.secondary_opts) + return frozenset(opts) + + +def _split_argv(argv: list[str], command: str) -> tuple[list[str], list[str]] | None: + """Split argv into (root options, command arguments), or ``None``.""" + value_opts = _root_value_opts() + i = 0 + while i < len(argv): + token = argv[i] + if token == command: + return argv[:i], argv[i + 1 :] + if token == "--" or not token.startswith("-"): + return None + if token in value_opts: + i += 2 + continue + if not token.startswith("--"): + # A cluster such as -qo: a value letter takes the rest of the + # token, or the next one when it is last. + for j, letter in enumerate(token[1:]): + if f"-{letter}" in value_opts: + if j == len(token) - 2: + i += 1 + break + i += 1 + return None + + +def _drop_output(args: list[str]) -> list[str]: + """``args`` without -o/--output and its value.""" + kept: list[str] = [] + i = 0 + while i < len(args): + token = args[i] + if token == "--": + kept.extend(args[i:]) + break + if token in ("-o", "--output"): + i += 2 + continue + if token.startswith("--output=") or ( + token.startswith("-o") and not token.startswith("--") + ): + i += 1 + continue + kept.append(token) + i += 1 + return kept + + +def _secret_bearing(tokens: list[str]) -> bool: + """True when argv holds a key or a URL with userinfo or a query.""" + for token in tokens: + if token in ("--api-key", "-k") or token.startswith("--api-key="): + return True + if token.startswith("-k") and not token.startswith("--"): + return True + if "://" in token: + try: + parts = urlsplit(token) + except ValueError: + return True + if parts.username or parts.password or parts.query: + return True + return False + + +def _next_step(tokens: list[str]) -> str | None: + """A command line runnable verbatim, or ``None``.""" + if _secret_bearing(tokens): + return None + for token in tokens: + if token.startswith("=") or ( + token.startswith("<") and token.endswith(">") and token[1:-1].isupper() + ): + return None + line = shlex.join(tokens) + if os.name == "nt" and line != " ".join(tokens): + return None + if shlex.join(shlex.split(line)) != line or tokens[:1] == ["sudo"]: + return None + return line + + +def _proposed_out_file( + *, is_flux: bool, encoding: str | None, container: str | None +) -> str: + """A new file name in the cwd for next_step: never an existing file.""" + ext = _play_suffix(is_flux=is_flux, encoding=encoding, container=container) + while True: + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + name = f"speech-{stamp}-{secrets.token_hex(2)}{ext}" + if not os.path.lexists(name): + return name + + +def _deprecate_output() -> None: + """The one DEPRECATED line for speak -o/--output FILE, even under -q.""" + global _deprecation_shown + if _deprecation_shown: + return + _deprecation_shown = True + text = msg.D_SPEAK_OUTPUT.format(prog=_prog()) + if _agentic: + stderr_console.print(f"DEPRECATED: {escape(text)}", soft_wrap=True) + else: + stderr_console.print( + f"[yellow]⚠[/yellow] Deprecated: {escape(text)}", soft_wrap=True + ) + + +class _Terminated(BaseException): + """SIGTERM or SIGHUP arrived mid-stream: unwind so a partial file + or temp file is removed, then end by that signal.""" + + def __init__(self, signum: int) -> None: + super().__init__(signum) + self.signum = signum + + +#: How long a signalled run may take to unwind before the signal is +#: delivered again and ends it. This timer is how a Flux run ends after a +#: signal when the server doesn't answer the closing handshake: the SDK +#: waits up to 10 s for it in the main thread. A close still waiting in +#: ``finally`` is cut short the same way. SIGTERM and SIGHUP then end at +#: once; Ctrl-C runs the usual cancel path, so it ends a little later. +#: Both stay well inside the 2.6 s a signalled run may take to exit. +_SIGNAL_GRACE_S = 1.5 + + +def _end_by_signal(signum: int) -> None: + """End the process by ``signum`` (POSIX); on Windows, exit 128+N.""" + if os.name == "nt": + raise SystemExit(128 + signum) + _die_by_signal(signum) + + +@contextmanager +def _terminate_cleanly(cleanup: Callable[[], None]) -> Iterator[None]: + """For the duration of a request and stream: + + - SIGTERM and SIGHUP raise :class:`_Terminated` in the main thread + instead of killing the process before ``cleanup`` can run; + - SIGINT raises ``KeyboardInterrupt`` as before. + + Unwinding can stall in a WebSocket closing handshake the server doesn't + answer. So the first signal also arms a timer: if the run is still here + after ``_SIGNAL_GRACE_S``, the signal is delivered again, and a second + signal runs ``cleanup`` and ends the run: SIGINT by ``KeyboardInterrupt`` + again (the usual cancel message, exit 2, exit handlers run; a hard exit + follows only if that unwind stalls too), SIGTERM and SIGHUP by the signal + itself. A double Ctrl-C after the guard has exited is the default + handler's: exit by SIGINT (130), as before. + + Signals that arrive during a file critical section are deferred (see + ``sinks.critical_section``); the most severe is delivered after it. + + A signal is left exactly as it was when its handler isn't ours to + replace: + + - ``SIG_IGN`` (nohup ignores SIGHUP, a background job in a script starts + with SIGINT ignored, ``trap '' TERM``): the run keeps going, as 0.4.0 + did, and the file is completed; + - a handler not installed from Python (``signal.getsignal`` returns + ``None``): ``signal.signal`` can't put it back afterwards, so it is + not replaced at all. + + On exit the guard is closed first (its handler then only records a + signal, and the timer sends nothing more), the signals are blocked where + ``pthread_sigmask`` exists, the previous handlers are restored and the + timers cancelled, the mask is restored, and a signal recorded meanwhile + is raised again through the restored handler. The guard is closed before + the mask call because that call itself runs pending handlers. + """ + if threading.current_thread() is not threading.main_thread(): + yield + return + first: list[int] = [] + timers: list[threading.Timer] = [] + closed: list[bool] = [] + late: list[int] = [] + sent: list[int] = [] # re-deliveries the timer made + lock = threading.Lock() + + def redeliver(signum: int) -> None: + # Under the lock, so nothing is sent once the guard is closing. + with lock: + if closed: + return + sent.append(signum) + _send_to_main(signum) + + def arm(delay: float, action: Callable[..., None], *args: Any) -> None: + timer = threading.Timer(delay, action, args) + timer.daemon = True + timer.start() + timers.append(timer) + + @_coalescing # a signal while this runs is coalesced (core, as for main) + def handler(signum: int, _frame: Any) -> None: + if signum in sent: + sent.remove(signum) + if closed: + return # our own re-delivery, arriving late: drop it + elif closed: + late.append(signum) + return + if _defer_in_importlib(signum, _frame): + return # raising inside importlib could leave its lock held + if sinks.defer_signal(signum): + return # creating, committing or removing the file + if first: + # Second signal, or the timer: stop waiting for the stream. + cleanup() + decisive = sinks.most_severe([first[0], signum]) or signum + if decisive == signal.SIGINT: + # Unwind again from wherever the stall is; if that stalls + # too, end without waiting any longer. + arm(_SIGNAL_GRACE_S, os._exit, 2) + raise KeyboardInterrupt + _end_by_signal(decisive) + first.append(signum) + arm(_SIGNAL_GRACE_S, redeliver, signum) + if signum == signal.SIGINT: + raise KeyboardInterrupt + raise _Terminated(signum) + + handler._deepctl_defers = True # type: ignore[attr-defined] + + previous: dict[int, Any] = {} + for name in ("SIGINT", "SIGTERM", "SIGHUP"): + sig = getattr(signal, name, None) + if sig is None: + continue + current = signal.getsignal(sig) + if current is signal.SIG_IGN or current is None: + continue + try: + signal.signal(sig, handler) + except (OSError, ValueError): + continue + previous[sig] = current + try: + yield + finally: + with lock: + closed.append(True) + blocked = None + if hasattr(signal, "pthread_sigmask"): + with suppress(OSError, ValueError): + blocked = signal.pthread_sigmask(signal.SIG_BLOCK, set(previous)) + try: + sinks.restore_handlers(previous) + finally: + for timer in timers: + timer.cancel() + if blocked is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, blocked) + if late: + again = sinks.most_severe(late) + late.clear() + if again is not None: + signal.raise_signal(again) + + +@dataclass +class _Plan: + """What the pre-request checks decided.""" + + target: sinks.Target | None = None + + def _fail(message: str) -> BaseResult: """An error result whose message the user actually sees. @@ -472,6 +795,25 @@ def timing(self) -> str: return f"first audio {fa}, {time.monotonic() - self._start:.1f}s total" +def _speak_result( + audio_file: sinks.AudioFile | None, + *, + model: str, + bytes_written: int, + played: bool = False, +) -> SpeakResult: + """The 0.4.0 ``SpeakResult`` (unchanged keys, including ``played``).""" + path = audio_file.path if audio_file is not None else "" + return SpeakResult( + status="success", + message=f"Audio saved to {path}" if path else f"Played {bytes_written:,} bytes", + output_path=path, + model=model, + bytes_written=bytes_written, + played=played, + ) + + class SpeakCommand(BaseCommand): """Command for generating speech from text using Deepgram TTS.""" @@ -491,26 +833,27 @@ class SpeakCommand(BaseCommand): 'dg speak "Hello from Deepgram" --play', "dg speak --list-voices", 'dg speak "Hello world"', - 'dg speak "Hello world" -o hello.wav', - 'dg speak "Hello world" -o hello.wav --play', - "dg speak --file message.txt -o output.wav", + 'dg speak "Hello world" --out-file hello.wav', + 'dg speak "Hello world" --out-file hello.wav --play', + "dg speak --file message.txt --out-file output.wav", 'dg speak "Hello" | ffplay -loglevel error -nodisp -autoexit -', # Flux TTS streaming controls: --speed (0.85-1.15) and beta # --expressivity (-2..2, default 0). - 'dg speak "A little slower, please" --speed 0.9 -o slow.wav', - 'dg speak "So exciting!" --expressivity 2 -o lively.wav # beta; default: 0', + 'dg speak "A little slower, please" --speed 0.9 --out-file slow.wav', + 'dg speak "So exciting!" --expressivity 2 --out-file lively.wav # beta; default: 0', # Aura (Speak v1, batch REST) — opt in with -m aura-*; needed for # containerized formats like mp3. - 'dg speak "Hello" -m aura-2-asteria-en -o hello.mp3', - 'dg speak "Hello" -m aura-2-luna-en -o hello.wav --encoding linear16 --container wav', - 'echo "Hello" | dg speak -o hello.mp3 -m aura-2-asteria-en', + 'dg speak "Hello" -m aura-2-asteria-en --out-file hello.mp3', + 'dg speak "Hello" -m aura-2-luna-en --out-file hello.wav --encoding linear16 --container wav', + 'echo "Hello" | dg speak --out-file hello.mp3 -m aura-2-asteria-en', # Aura-2 Spanish voice (run `dg models` for the full list). - 'dg speak "Hola, mundo" -m aura-2-selena-es -o hola.mp3', + 'dg speak "Hola, mundo" -m aura-2-selena-es --out-file hola.mp3', ] agent_help = ( "Convert text to speech using Deepgram's TTS API. " "Text can be provided as an argument, from a file, or piped via stdin. " - "Audio is written to a file (--output) or stdout for piping. " + "Audio is written to a file (--out-file; add --force to replace an " + "existing file) or stdout for piping. -o/--output FILE is deprecated. " "By default (flux-* models, e.g. flux-alexis-en) uses Flux TTS / Speak " "v2, streaming over WebSocket and emitting raw audio; linear16 output is " "wrapped in a WAV container so it is directly playable. Pass an aura-* " @@ -531,10 +874,30 @@ def get_arguments(self) -> list[dict[str, Any]]: "required": False, "default": None, }, + { + "names": ["--out-file"], + "help": ( + "Save the audio to FILE. An existing file isn't " + "overwritten unless you add --force." + ), + "type": str, + "metavar": "FILE", + "is_option": True, + }, + { + "names": ["--force"], + "help": "With --out-file, overwrite FILE if it already exists.", + "is_flag": True, + "is_option": True, + }, { "names": ["--output", "-o"], - "help": "Output file path (required if stdout is a terminal)", + "help": ( + "Deprecated: use --out-file. Saves the audio to FILE and " + "overwrites it. In 1.0, -o will set the output format." + ), "type": str, + "metavar": "FILE", "is_option": True, }, { @@ -603,7 +966,8 @@ def get_arguments(self) -> list[dict[str, Any]]: "names": ["--play"], "help": ( "Play the audio through your system player (ffplay, afplay, " - "paplay, or aplay). Can be combined with -o to save and play." + "paplay, or aplay). Can be combined with --out-file to save " + "and play." ), "is_flag": True, "is_option": True, @@ -616,6 +980,124 @@ def get_arguments(self) -> list[dict[str, Any]]: }, ] + # ── Checks before the request ───────────────────────────────────── + + _plan: _Plan | None = None + + def pre_auth_check(self, config: Config, **kwargs: Any) -> Any | None: + """Refusals that need no request: they run + before authentication, so nothing is sent or billed.""" + self._plan = None + planned = self._preflight(kwargs) + if isinstance(planned, SpeakCodedError): + return planned + self._plan = planned + return None + + def _preflight(self, kwargs: dict[str, Any]) -> _Plan | SpeakCodedError: + if kwargs.get("list_voices"): + return _Plan() + out_file = kwargs.get("out_file") or None + legacy = kwargs.get("output") or None + prog = _prog() + + if out_file and legacy: + return _coded( + msg.E_SPEAK_OUT_CONFLICT, + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_SPEAK_OUT_CONFLICT, + hint_agent=msg.H_SPEAK_OUT_CONFLICT, + next_step=_next_step([prog, self.name, "--help"]), + ) + if legacy and legacy.lower() == "text": + # Not a root format either: point at --out-file instead. + return _coded( + msg.E_SPEAK_O_TEXT.format(prog=prog), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_SPEAK_O_TEXT, + hint_agent=msg.H_SPEAK_O_TEXT, + next_step=self._o_format_next_step(prog, legacy, kwargs), + ) + if legacy and legacy.lower() in _FORMAT_WORDS: + return _coded( + msg.E_SPEAK_O_FORMAT.format(prog=prog, value=legacy), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_SPEAK_O_FORMAT.format(prog=prog, value=legacy), + hint_agent=msg.H_SPEAK_O_FORMAT.format(prog=prog, value=legacy), + next_step=self._o_format_next_step(prog, legacy, kwargs), + ) + if legacy: + _deprecate_output() + + path = out_file or legacy + if not path: + return _Plan() + planned = sinks.plan_target( + path, overwrite=bool(legacy) or bool(kwargs.get("force")) + ) + if isinstance(planned, sinks.Refusal): + if planned.reason == "exists": + return _coded( + msg.E_OUT_FILE_EXISTS.format(out_file=path, prog=prog), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_OUT_FILE_EXISTS, + hint_agent=msg.H_OUT_FILE_EXISTS, + ) + assert planned.error is not None + return _output_unwritable(planned.error, path=path) + if ( + planned.kind == "stdout" + and get_output_format() in ("json", "yaml") + and format_source() == "explicit" + ): + # The audio is stdout's payload; there's no room for JSON. + fmt = get_output_format().upper() + return _coded( + msg.E_STDOUT_BYTES.format( + prog=prog, command_path=self.name, format=fmt + ), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_STDOUT_BYTES.format(format=fmt), + hint_agent=msg.H_STDOUT_BYTES.format(format=fmt), + next_step=_next_step([prog, self.name, "--help"]), + ) + return _Plan(target=planned) + + def _o_format_next_step( + self, prog: str, value: str, kwargs: dict[str, Any] + ) -> str | None: + """``{prog} {root_options} -o {value} speak {speak_args} --out-file + {out_file}``, or ``None`` when it can't be built safely. For + ``-o text``, which no command takes, the ``-o`` is dropped.""" + text = value.lower() == "text" + if not text and value.lower() not in _ROOT_FORMATS: + return None + ctx = click.get_current_context(silent=True) + name = (ctx.info_name if ctx is not None else None) or self.name + split = _split_argv(_argv(), name) + if split is None: + return None + root, args = split + model = kwargs.get("model") or _DEFAULT_MODEL + out_file = _proposed_out_file( + is_flux=model.lower().startswith("flux-"), + encoding=kwargs.get("encoding"), + container=kwargs.get("container"), + ) + return _next_step( + [ + prog, + *_drop_output(root), + *(() if text else ("-o", value)), + name, + *_drop_output(args), + "--out-file", + out_file, + ] + ) + + # ── Run ─────────────────────────────────────────────────────────── + def handle( self, config: Config, @@ -623,8 +1105,12 @@ def handle( client: DeepgramClient, **kwargs: Any, ) -> BaseResult | SpeakCodedError | None: + planned = self._plan if self._plan is not None else self._preflight(kwargs) + self._plan = None + if isinstance(planned, SpeakCodedError): + return planned + text = kwargs.get("text") - output_path = kwargs.get("output") model = kwargs.get("model") or _DEFAULT_MODEL encoding = kwargs.get("encoding") container = kwargs.get("container") @@ -634,6 +1120,7 @@ def handle( file_path = kwargs.get("file") play = kwargs.get("play", False) list_voices = kwargs.get("list_voices", False) + output_path = planned.target.path if planned.target else None # --list-voices: discover voices and exit before any TTS generation. if list_voices: @@ -661,17 +1148,15 @@ def handle( player = _find_audio_player() if player is None: return _fail( - "No audio player found — install ffmpeg, or use -o to save a file." + "No audio player found. Install ffmpeg, or save the audio " + "with --out-file FILE." ) # If stdout is a TTY and there's nowhere for the audio to go, require an # explicit destination. --play satisfies that requirement. stdout_is_tty = sys.stdout.isatty() if not output_path and not play and stdout_is_tty: - return _fail( - "No output specified. Use -o/--output to save to a file, " - "--play to hear it, or pipe stdout." - ) + return _fail(msg.NO_OUTPUT_TTY) # Only the documented flux-* namespace uses speak.v2. Aura and unknown # model names pass through to the REST API so the service can resolve them. @@ -711,234 +1196,362 @@ def handle( # Playing sends the audio to the player, not to stdout, so a # redirect or pipe would otherwise collect nothing and give no # hint why. - print_warning( - "--play sends the audio to your player, so stdout stays empty. " - "Add -o to save a file too.", - stderr=True, - ignore_quiet=True, + print_warning(msg.PLAY_STDOUT_EMPTY, stderr=True, ignore_quiet=True) + + audio_file = sinks.AudioFile(planned.target) if planned.target else None + if audio_file is not None and audio_file.is_stdout: + # stdout carries audio, so a failure is reported on stderr + # only, never as JSON after the audio. + self.stdout_is_protocol = True + try: + with _terminate_cleanly( + audio_file.abort if audio_file is not None else lambda: None + ): + try: + result = self._speak( + client, + is_flux=is_flux, + text=text, + model=model, + encoding=encoding, + container=container, + sample_rate=sample_rate, + speed=speed, + expressivity=expressivity, + player=player, + audio_file=audio_file, + ) + except BaseException: + # Clean up while the guard is still installed, so a + # second signal can't cut the removal short. + if audio_file is not None: + audio_file.abort() + raise + except _Terminated as exc: + # The partial file or temp file is gone (an old file is kept); + # end by the signal itself. + _end_by_signal(exc.signum) + raise + except sinks.Exists: + # O_EXCL lost a race with another writer: nothing of ours exists. + return _coded( + msg.E_OUT_FILE_EXISTS.format(out_file=output_path, prog=_prog()), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_OUT_FILE_EXISTS, + hint_agent=msg.H_OUT_FILE_EXISTS, ) + except sinks.WriteFailed as exc: + if audio_file is not None: + audio_file.abort() + if exc.stdout: + return _stdout_unwritable(exc.error) + assert audio_file is not None + return _output_unwritable(exc.error, path=audio_file.path) + except sinks.ReaderGone: + # Whoever read the pipe went away: stop quietly, like stdout. + if audio_file is not None and audio_file.is_stdout: + _silence_stdout() + return None + finally: + if audio_file is not None: + audio_file.abort() # a no-op once the file was committed + + if ( + audio_file is not None + and audio_file.is_stdout + and isinstance(result, SpeakResult) + and result.status == "success" + ): + # The audio was stdout's payload, so the summary goes to + # stderr and no document follows it. + n = audio_file.bytes_written + if _agentic: + console.print(f"OK: Wrote {n:,} bytes to stdout", markup=False) + else: + console.print(f"[green]✓ Wrote {n:,} bytes to stdout[/green]") + return None + return result + def _speak( + self, + client: DeepgramClient, + *, + is_flux: bool, + text: str, + model: str, + encoding: str | None, + container: str | None, + sample_rate: float | None, + speed: float | None, + expressivity: int | None, + player: str | None, + audio_file: sinks.AudioFile | None, + ) -> BaseResult | SpeakCodedError | None: if is_flux: - # WebSocket streaming path. Streaming output is raw audio, so - # default to linear16 @ 24kHz and wrap it in WAV for playback. - # These paths raise (rather than returning an error result) so a - # failure exits non-zero and is visible in every output format — - # a returned error is only printed in structured output and still - # exits 0. - eff_encoding = encoding or "linear16" - if eff_encoding not in ("linear16", "mulaw", "alaw"): - raise click.ClickException( - f"Encoding '{eff_encoding}' is not supported for Flux " - "(Speak v2) streaming, which emits raw audio. Use " - "linear16 (default), mulaw, or alaw." - ) - eff_sample_rate = sample_rate or 24000.0 - - # A live spinner + byte counter + time-to-first-audio on stderr, so - # the stream is visibly arriving rather than a single static line. - prog = _StreamProgress(model) - stream = prog.track( - client.speak_text_stream( - text=text, - model=model, - encoding=eff_encoding, - sample_rate=eff_sample_rate, - speed=speed, - expressivity=expressivity, - ) + return self._speak_flux( + client, + text=text, + model=model, + encoding=encoding, + sample_rate=sample_rate, + speed=speed, + expressivity=expressivity, + player=player, + audio_file=audio_file, ) + return self._speak_v1( + client, + text=text, + model=model, + encoding=encoding, + container=container, + sample_rate=sample_rate, + player=player, + audio_file=audio_file, + ) - # Flux linear16 into a stdin-reading player: stream it. The - # player gets the streaming WAV header before the first frame and - # then every frame as Flux emits it, so sound starts at - # first-audio latency instead of after the whole utterance — the - # same framing `dg speak | ffplay -` relies on. - if ( - player is not None - and player in _STDIN_PLAYER_ARGV - and eff_encoding == "linear16" - ): - console.print(f"[blue]Playing audio ({player})...[/blue]") - pcm = bytearray() - saved_bytes = 0 - player_gone = False - try: - with _player_stdin(player) as sink: - wrote_header = False - for chunk in stream: - if not chunk: - continue - if output_path: - pcm.extend(chunk) - if player_gone: - continue - try: - if not wrote_header: - sink.write( - _streaming_wav_header( - sample_rate=int(eff_sample_rate) - ) - ) - wrote_header = True - sink.write(chunk) - sink.flush() - except BrokenPipeError: - # The player exited first (ffplay's "q", for - # instance). That is the user stopping - # playback, not a failure. With -o, keep - # draining so the file is still complete; - # otherwise stop and close the stream. - player_gone = True - if not output_path: - break - - if output_path and pcm: - # Save before waiting on the player (that wait - # happens on leaving this block), so stopping - # playback still leaves the file behind. The file - # gets a real, exact-length header rather than the - # streaming placeholder the player was handed. - audio_bytes = _pcm_to_wav( - bytes(pcm), sample_rate=int(eff_sample_rate) - ) - Path(output_path).write_bytes(audio_bytes) - saved_bytes = len(audio_bytes) - except click.ClickException: - raise - except Exception as e: - raise click.ClickException(f"Flux streaming failed: {e}") - finally: - _close(stream) - - if prog.total == 0: - raise click.ClickException( - "Flux (Speak v2) streaming returned no audio." - ) + # ── Flux (Speak v2, WebSocket streaming) ────────────────────────── - if output_path: - console.print( - f"[green]Audio saved to {output_path}[/green] " - f"({saved_bytes:,} bytes — {prog.timing()})" - ) - else: - console.print( - f"[green]✓ Played {prog.total:,} bytes[/green] " - f"({prog.timing()})" - ) - return SpeakResult( - status="success", - message=( - f"Audio saved to {output_path}" - if output_path - else f"Played {prog.total:,} bytes" - ), - output_path=output_path or "", + def _speak_flux( + self, + client: DeepgramClient, + *, + text: str, + model: str, + encoding: str | None, + sample_rate: float | None, + speed: float | None, + expressivity: int | None, + player: str | None, + audio_file: sinks.AudioFile | None, + ) -> BaseResult | SpeakCodedError | None: + # Streaming output is raw audio, so default to linear16 @ 24kHz and + # wrap it in WAV for playback. These paths raise (rather than + # returning an error result) so a failure exits non-zero and is + # visible in every output format. + eff_encoding = encoding or "linear16" + if eff_encoding not in ("linear16", "mulaw", "alaw"): + raise click.ClickException( + f"Flux voices stream raw audio, so --encoding {eff_encoding} " + "isn't available. Use linear16 (the default), mulaw or alaw." + ) + rate = int(sample_rate or 24000.0) + + # A live byte counter + time-to-first-audio on stderr, so the stream + # is visibly arriving rather than a single static line. + prog = _StreamProgress(model) + raw = client.speak_text_stream( + text=text, + model=model, + encoding=eff_encoding, + sample_rate=float(rate), + speed=speed, + expressivity=expressivity, + ) + stream = prog.track(raw) + try: + if player in _STDIN_PLAYER_ARGV and eff_encoding == "linear16": + assert player is not None + return self._flux_play_streaming( + stream, + prog, + player=player, + rate=rate, + audio_file=audio_file, model=model, - bytes_written=saved_bytes or prog.total, - played=True, ) + if audio_file is not None or player: + return self._flux_buffered( + stream, + prog, + player=player, + rate=rate, + encoding=eff_encoding, + audio_file=audio_file, + model=model, + ) + return self._flux_to_stdout( + stream, + prog, + header=( + _streaming_wav_header(sample_rate=rate) + if eff_encoding == "linear16" + else b"" + ), + ) + except click.ClickException: + raise + except _SINK_ERRORS: + raise + except Exception as e: + raise click.ClickException(f"Flux streaming failed: {e}") + finally: + _close(stream, raw) - if output_path or player: - # Saving, or playing through afplay (which has no stdin mode), - # needs the full utterance: a WAV file must declare its data - # length in the header (known only once the stream ends), and - # afplay is handed a complete file. So buffer, then wrap once - # and reuse for both sinks. - pcm = bytearray() + def _flux_play_streaming( + self, + stream: Iterator[bytes], + prog: _StreamProgress, + *, + player: str, + rate: int, + audio_file: sinks.AudioFile | None, + model: str, + ) -> BaseResult: + """Flux linear16 into a stdin-reading player, frame by frame. + + The player gets the streaming WAV header before the first frame and + then every frame as Flux emits it, so sound starts at first-audio + latency — the same framing `dg speak | ffplay -` relies on. A file + gets the exact-length WAV once the stream ends; it is created on + the first frame. + """ + console.print(f"[blue]Playing audio ({player})...[/blue]") + pcm = bytearray() + header = _streaming_wav_header(sample_rate=rate) + player_gone = False + with _player_stdin(player) as sink: + for chunk in stream: + if not chunk: + continue + if audio_file is not None: + audio_file.open() + pcm.extend(chunk) + if player_gone: + continue try: - for chunk in stream: - pcm.extend(chunk) - except click.ClickException: - raise - except Exception as e: - raise click.ClickException(f"Flux streaming failed: {e}") - - if not pcm: - # No audio means the stream errored upstream or returned - # nothing. Wrapping empty PCM yields a valid header-only - # WAV, so guard rather than report success on a silent file. - raise click.ClickException( - "Flux (Speak v2) streaming returned no audio." - ) + if header: + sink.write(header) + header = b"" + sink.write(chunk) + sink.flush() + except BrokenPipeError: + # The player exited first (ffplay's "q"): the user + # stopped playback, not a failure. With a file, keep + # draining so it is complete; otherwise stop. + player_gone = True + if audio_file is None: + break + + if audio_file is not None and pcm: + # Save before waiting on the player (on leaving this block), + # so stopping playback still leaves the file behind. + audio_file.write(_pcm_to_wav(bytes(pcm), sample_rate=rate)) + audio_file.commit() + + if prog.total == 0: + raise click.ClickException( + "Deepgram returned no audio. Try again, or try another --model." + ) - if eff_encoding == "linear16": - audio_bytes = _pcm_to_wav( - bytes(pcm), sample_rate=int(eff_sample_rate) - ) - else: - audio_bytes = bytes(pcm) - - total_bytes = len(audio_bytes) - if output_path: - Path(output_path).write_bytes(audio_bytes) - console.print( - f"[green]Audio saved to {output_path}[/green] " - f"({total_bytes:,} bytes — {prog.timing()})" - ) + if audio_file is not None: + self._saved(audio_file, f"{prog.timing()}") + else: + console.print( + f"[green]✓ Played {prog.total:,} bytes[/green] ({prog.timing()})" + ) + return _speak_result( + audio_file, + model=model, + bytes_written=(audio_file.bytes_written if audio_file else 0) or prog.total, + played=True, + ) - if player: - suffix = _play_suffix( - is_flux=True, encoding=eff_encoding, container=None - ) - console.print(f"[blue]Playing audio ({player})...[/blue]") - _play_audio(player, audio_bytes, suffix=suffix) - - return SpeakResult( - status="success", - message=( - f"Audio saved to {output_path}" - if output_path - else f"Played {total_bytes:,} bytes" - ), - output_path=output_path or "", - model=model, - bytes_written=total_bytes, - played=bool(player), - ) + def _flux_buffered( + self, + stream: Iterator[bytes], + prog: _StreamProgress, + *, + player: str | None, + rate: int, + encoding: str, + audio_file: sinks.AudioFile | None, + model: str, + ) -> BaseResult: + """Flux into a file and/or afplay: both need the whole utterance. - # Pipe path — write each chunk to stdout as it arrives so a - # downstream player starts on the first frame (the point of - # "streaming by default"). For linear16 we emit a streaming WAV - # header before the first frame so `| ffplay -` plays with no - # extra flags; raw mulaw/alaw stream as-is. - header = ( - _streaming_wav_header(sample_rate=int(eff_sample_rate)) - if eff_encoding == "linear16" - else b"" + A WAV file declares its data length in the header, known only once + the stream ends, and afplay is handed a complete file. The file is + still created when the first frame arrives. + """ + pcm = bytearray() + for chunk in stream: + if chunk and audio_file is not None: + audio_file.open() + pcm.extend(chunk) + + if not pcm: + # Wrapping empty PCM yields a valid header-only WAV, so guard + # rather than report success on a silent file. + raise click.ClickException( + "Deepgram returned no audio. Try again, or try another --model." ) - try: - _, outcome = self._stream_to_stdout(stream, header=header) - except click.ClickException: - raise - except Exception as e: - raise click.ClickException(f"Flux streaming failed: {e}") - finally: - _close(stream) - if outcome is not True: - # The reader went away (quiet, exit 0) or stdout failed. - return outcome or None - if prog.total == 0: - # Guard before anything is written: the header is only emitted - # on the first frame, so a no-audio stream writes nothing. - raise click.ClickException( - "Flux (Speak v2) streaming returned no audio." - ) + if encoding == "linear16": + audio_bytes = _pcm_to_wav(bytes(pcm), sample_rate=rate) + else: + audio_bytes = bytes(pcm) - console.print( - f"[green]✓ Streamed {prog.total:,} bytes to stdout[/green] " - f"({prog.timing()})" + if audio_file is not None: + audio_file.write(audio_bytes) + audio_file.commit() + self._saved(audio_file, prog.timing()) + + if player: + suffix = _play_suffix(is_flux=True, encoding=encoding, container=None) + console.print(f"[blue]Playing audio ({player})...[/blue]") + _play_audio(player, audio_bytes, suffix=suffix) + + return _speak_result( + audio_file, model=model, bytes_written=len(audio_bytes), played=bool(player) + ) + + def _flux_to_stdout( + self, stream: Iterator[bytes], prog: _StreamProgress, *, header: bytes + ) -> SpeakCodedError | None: + """Pipe path: each frame goes to stdout as it arrives, after the + streaming WAV header for linear16 (so `| ffplay -` plays with no + extra flags); raw mulaw/alaw stream as-is.""" + _, outcome = self._stream_to_stdout(stream, header=header) + if outcome is not True: + # The reader went away (quiet, exit 0) or stdout failed. + return outcome or None + + if prog.total == 0: + # The header is only emitted on the first frame, so a no-audio + # stream wrote nothing. + raise click.ClickException( + "Deepgram returned no audio. Try again, or try another --model." ) - # Return None so the framework skips output_result: in agentic/CI - # (or --output json) mode it would serialize the result to the - # stdout console — i.e. append JSON to the audio we just streamed, - # corrupting a `> out.wav` redirect. The summary above already went - # to the stderr console. - return None - # REST path (Speak v1, including Aura and unknown-model pass-through). - # Each chunk goes on as the SDK yields it: to stdout (flushed), or to - # a stdin-reading player, so neither waits for the whole utterance. + console.print( + f"[green]✓ Streamed {prog.total:,} bytes to stdout[/green] " + f"({prog.timing()})" + ) + # Return None so the framework skips output_result: in agentic/CI + # (or --output json) mode it would serialize the result to the + # stdout console — i.e. append JSON to the audio we just streamed, + # corrupting a `> out.wav` redirect. + return None + + # ── Speak v1 (REST) ─────────────────────────────────────────────── + + def _speak_v1( + self, + client: DeepgramClient, + *, + text: str, + model: str, + encoding: str | None, + container: str | None, + sample_rate: float | None, + player: str | None, + audio_file: sinks.AudioFile | None, + ) -> BaseResult | SpeakCodedError | None: + """Speak v1, including Aura and unknown-model pass-through. Each + chunk goes on as the SDK yields it: to stdout (flushed), to a + stdin-reading player, and to the file, so none of them waits for + the whole utterance.""" console.print(f"[blue]Generating speech with {model}...[/blue]") try: audio_iter = client.speak_text( @@ -953,34 +1566,30 @@ def handle( try: if player in _STDIN_PLAYER_ARGV: + assert player is not None return self._v1_play_streaming( - audio_iter, player=player, output_path=output_path, model=model + audio_iter, player=player, audio_file=audio_file, model=model ) if player: return self._v1_play_buffered( audio_iter, player=player, - output_path=output_path, + audio_file=audio_file, model=model, encoding=encoding, container=container, ) - if output_path: - total_bytes = 0 - with open(Path(output_path), "wb") as f: - for chunk in audio_iter: - f.write(chunk) - total_bytes += len(chunk) - - console.print( - f"[green]Audio saved to {output_path}[/green] ({total_bytes:,} bytes)" - ) - return SpeakResult( - status="success", - message=f"Audio saved to {output_path}", - output_path=output_path, - model=model, - bytes_written=total_bytes, + if audio_file is not None: + for chunk in audio_iter: + audio_file.write(chunk) + if not audio_file.bytes_written: + return _fail( + "Deepgram returned no audio. Try again, or try another --model." + ) + audio_file.commit() + self._saved(audio_file) + return _speak_result( + audio_file, model=model, bytes_written=audio_file.bytes_written ) total_bytes, outcome = self._stream_to_stdout(audio_iter) @@ -990,12 +1599,13 @@ def handle( console.print(f"[green]✓ Wrote {total_bytes:,} bytes to stdout[/green]") # Return None so the framework skips output_result — otherwise # agentic/CI (or --output json) mode serializes JSON onto the - # same stdout as the audio, corrupting a `> out` redirect. The - # summary above went to the stderr console. + # same stdout as the audio, corrupting a `> out` redirect. return None except click.ClickException: raise + except _SINK_ERRORS: + raise except Exception as e: # Raise (not return an error result) so the failure exits non-zero # in every output mode — a returned BaseResult is only printed and @@ -1044,109 +1654,101 @@ def _v1_play_streaming( audio_iter: Iterator[bytes], *, player: str, - output_path: str | None, + audio_file: sinks.AudioFile | None, model: str, ) -> BaseResult: """Speak v1 into a stdin-reading player, chunk by chunk. The player is started before the request, so sound starts as the - first chunk arrives. With -o, every chunk also goes to the file. + first chunk arrives. With a file, every chunk also goes there. """ console.print(f"[blue]Playing audio ({player})...[/blue]") total = 0 player_gone = False try: - with ( - _player_stdin(player) as sink, - open(Path(output_path), "wb") - if output_path - else nullcontext() as saved, - ): + with _player_stdin(player) as sink: for chunk in audio_iter: if not chunk: continue total += len(chunk) - if saved is not None: - saved.write(chunk) + if audio_file is not None: + audio_file.write(chunk) if player_gone: continue try: sink.write(chunk) sink.flush() except BrokenPipeError: - # The user stopped playback (ffplay's "q"). With -o, - # keep draining so the file is complete; otherwise + # The user stopped playback (ffplay's "q"). With a + # file, keep draining so it is complete; otherwise # stop and close the response. player_gone = True - if saved is None: + if audio_file is None: break if not total: # Kill the player rather than hand it an empty stream. raise _NoAudio + if audio_file is not None: + # Before waiting on the player, so stopping playback + # still leaves the file behind. + audio_file.commit() except _NoAudio: - return _fail("TTS returned no audio.") - - if output_path: - console.print( - f"[green]Audio saved to {output_path}[/green] ({total:,} bytes)" + return _fail( + "Deepgram returned no audio. Try again, or try another --model." ) - return SpeakResult( - status="success", - message=( - f"Audio saved to {output_path}" - if output_path - else f"Played {total:,} bytes" - ), - output_path=output_path or "", - model=model, - bytes_written=total, - played=True, - ) + + if audio_file is not None: + self._saved(audio_file) + return _speak_result(audio_file, model=model, bytes_written=total, played=True) def _v1_play_buffered( self, audio_iter: Iterator[bytes], *, player: str, - output_path: str | None, + audio_file: sinks.AudioFile | None, model: str, encoding: str | None, container: str | None, ) -> BaseResult: """Speak v1 into a player that has no stdin mode (afplay). - It is handed a complete temp file, so the utterance is buffered. + It is handed a complete temp file, so the utterance is buffered; a + file still gets each chunk as it arrives. """ audio = bytearray() for chunk in audio_iter: + if audio_file is not None: + audio_file.write(chunk) audio.extend(chunk) audio_bytes = bytes(audio) - total_bytes = len(audio_bytes) if not audio_bytes: - return _fail("TTS returned no audio.") - - if output_path: - Path(output_path).write_bytes(audio_bytes) - console.print( - f"[green]Audio saved to {output_path}[/green] ({total_bytes:,} bytes)" + return _fail( + "Deepgram returned no audio. Try again, or try another --model." ) + if audio_file is not None: + audio_file.commit() + self._saved(audio_file) + suffix = _play_suffix(is_flux=False, encoding=encoding, container=container) console.print(f"[blue]Playing audio ({player})...[/blue]") _play_audio(player, audio_bytes, suffix=suffix) - return SpeakResult( - status="success", - message=( - f"Audio saved to {output_path}" - if output_path - else f"Played {total_bytes:,} bytes" - ), - output_path=output_path or "", - model=model, - bytes_written=total_bytes, - played=True, + return _speak_result( + audio_file, model=model, bytes_written=len(audio_bytes), played=True + ) + + @staticmethod + def _saved(audio_file: sinks.AudioFile, timing: str = "") -> None: + """The stderr summary for a saved file (none for stdout).""" + if audio_file.is_stdout: + return + extra = f" — {timing}" if timing else "" + console.print( + f"[green]Audio saved to {escape(audio_file.path)}[/green] " + f"({audio_file.bytes_written:,} bytes{extra})" ) # ── Result on stdout ────────────────────────────────────────────── diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py index 87f606a2..e732402d 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py @@ -8,6 +8,7 @@ # ── Catalogue ────────────────────────────────────────────────────────────── +CODE_INVALID_ARGUMENT = "invalid_argument" CODE_OUTPUT_UNWRITABLE = "output_unwritable" # output_unwritable: a write to stdout or the audio file failed. @@ -22,3 +23,57 @@ "user {prog} couldn't write its output ({reason}), and don't retry until " "it's fixed." ) + +# invalid_argument: --out-file is stdout and -o json/yaml was asked for. +# {format} is JSON or YAML. +E_STDOUT_BYTES = ( + "{prog} {command_path} would write the audio to stdout, so there's no room " + "for the {format} result." +) +H_STDOUT_BYTES = ( + "Give --out-file a different file from stdout, or don't ask for {format} output." +) + +# invalid_argument: --out-file names an existing file and --force is absent. +E_OUT_FILE_EXISTS = "{out_file} already exists; {prog} won't overwrite it." +H_OUT_FILE_EXISTS = "Choose a new file name, or pass --force to overwrite." + +# invalid_argument: speak -o given a bare output format. +E_SPEAK_O_FORMAT = ( + "{prog} speak -o is the audio file name, so -o {value} would write a file " + "named {value}." +) +H_SPEAK_O_FORMAT = ( + "To choose the output format, put -o before the command: " + "{prog} -o {value} speak TEXT --out-file FILE." +) + +# invalid_argument: speak -o text, which isn't a root format either. +E_SPEAK_O_TEXT = "{prog} speak -o text would write the audio to a file named text." +H_SPEAK_O_TEXT = ( + "Pass --out-file FILE to save the audio. There's no text output format." +) + +# invalid_argument: both -o/--output and --out-file. +E_SPEAK_OUT_CONFLICT = ( + "Use either --out-file or -o/--output for the audio file, not both." +) +H_SPEAK_OUT_CONFLICT = "Keep --out-file FILE and drop -o." + +# Deprecation notice for speak -o/--output FILE. Printed once per +# process, on stderr, even under --quiet. +D_SPEAK_OUTPUT = ( + "{prog} speak -o/--output FILE will set the output format in 1.0. " + "Use --out-file FILE instead (add --force to overwrite, as -o does)." +) + +# ── Legacy texts (0.4.x), now naming --out-file ──────────────────────────── + +NO_OUTPUT_TTY = ( + "No output specified. Use --out-file to save to a file, " + "--play to hear it, or pipe stdout." +) +PLAY_STDOUT_EMPTY = ( + "--play sends the audio to your player, so stdout stays empty. " + "Add --out-file to save a file too." +) diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py new file mode 100644 index 00000000..9440c99a --- /dev/null +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py @@ -0,0 +1,538 @@ +"""Where ``dg speak`` writes an audio file (``--out-file``, or the deprecated +``-o/--output``). + +Two halves: + +- :func:`plan_target` runs **before the request**. It ``lstat``s the path + and decides how the file will be written, or why it can't be: an + existing regular file or symlink without ``--force`` (``exists``); a + directory, or a parent that is missing or unwritable (``unwritable``). +- :class:`AudioFile` opens the file **when the first chunk arrives**, so a + failed request or an empty stream leaves nothing behind, and removes + whatever it created when the run fails. + +How a target is written (``Target.kind``): + +``new`` ``O_CREAT | O_EXCL``, mode 0666 & ~umask. ``O_EXCL`` is the + race guard behind the ``lstat`` check. +``replace`` ``--force`` (or ``-o``) over an existing file or symlink: a + 0600 temp file in the same directory, given the old file's + permission bits (never setuid, setgid or sticky; 0666 & ~umask + in place of a symlink) and ``os.replace``d + over the path at the end. A symlink is replaced, never written + through; a failed run leaves the old file. The result is a new + inode, so hard links to the old file keep the old bytes. +``special`` a character device or FIFO (``/dev/null``, a named pipe): + no ``O_EXCL``, no ``--force``. It is opened **before** the + request, non-blocking (a FIFO with no reader is refused, not + waited on), and the opened file must be the one that was + checked, so a path swapped for a regular file is never + written. Block devices and sockets are refused. +``stdout`` the same file as stdout (``--out-file /dev/stdout``, final + check): the audio goes to fd 1 and nothing else does. On + the first chunk fd 1 must still be the file that was checked; + the path is not opened again. + +Also refused before the request: the same file as stderr, where dg's own +messages go, and ``--force`` over a /dev/fd or /proc//fd entry. +""" + +from __future__ import annotations + +import errno +import os +import re +import signal +import stat +import sys +import threading +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@dataclass(frozen=True) +class Target: + path: str + kind: str # "new" | "replace" | "special" | "stdout" + #: ``replace`` over a regular file: its permission bits, kept. + mode: int | None = None + #: ``special``: the descriptor opened and checked before the request. + fd: int | None = None + #: ``stdout``: (st_dev, st_ino) of the file that was checked. + ident: tuple[int, int] | None = None + + +@dataclass(frozen=True) +class Refusal: + """Why a target can't be written, decided before the request.""" + + reason: str # "exists" | "unwritable" + error: OSError | None = None + + +class Exists(Exception): + """The file appeared between the check and the open (``O_EXCL``).""" + + +class WriteFailed(Exception): + """Writing the file (or stdout) failed; ``.error`` says why.""" + + def __init__(self, error: OSError, *, stdout: bool) -> None: + super().__init__(error) + self.error = error + self.stdout = stdout + + +class ReaderGone(Exception): + """The reader of a pipe went away (EPIPE).""" + + +def _stderr_stat() -> os.stat_result | None: + try: + return os.fstat(sys.stderr.fileno()) + except Exception: + return None + + +def stdout_stat() -> os.stat_result | None: + """``fstat`` of stdout, or ``None`` when it can't be read.""" + try: + return os.fstat(sys.stdout.fileno()) + except Exception: + return None + + +def _oserror(code: int, path: str) -> OSError: + return OSError(code, os.strerror(code), path) + + +def _read_umask() -> int: + """The process umask, read once when this module is imported. + + Linux publishes it in /proc. Elsewhere (macOS) the only call that reads + it also sets it, so ``os.umask`` is used only while this is the sole + thread, which it is when the command module loads. If another thread + exists, the umask is measured instead: a throwaway file is created + with mode 0o777 and the bits the kernel removed are the umask. + """ + try: + with open("/proc/self/status") as f: + for line in f: + if line.startswith("Umask:"): + return int(line.split()[1], 8) + except (OSError, ValueError, IndexError): + pass + if threading.active_count() == 1: + mask = os.umask(0o022) + os.umask(mask) + return mask + import tempfile + + probe = os.path.join(tempfile.gettempdir(), f".deepctl-umask-{os.getpid()}") + try: + fd = os.open(probe, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o777) + try: + return 0o777 & ~stat.S_IMODE(os.fstat(fd).st_mode) + finally: + os.close(fd) + os.unlink(probe) + except OSError: + return 0o022 + + +_UMASK = _read_umask() + +# The file types written as they are: character devices and FIFOs. +# These follow "Couldn't write the output to FILE: " (details.reason). +_NOT_SPECIAL = "it isn't a file, a pipe or a device like /dev/null" +_CHANGED = "something replaced it before writing started" +_STDERR = "it's stderr, which carries this command's messages" +_FD_PATH = "--force can't replace a file descriptor path" +_NO_READER = "no program is reading from that pipe" + +# Directories whose entries are open file descriptors: /dev/fd and +# /proc//fd. A temp file can't be created there, so --force can't +# replace what they name. +_FD_DIR = re.compile(r"/proc/(?:self|thread-self|\d+)(?:/task/\d+)?/fd") + +# ── Critical sections ─────────────────────────────────────────────────── +# Creating, committing and removing the file must not be cut short by a +# signal, or a temp file or a 0-byte file could be left behind. Inside a +# critical section (main thread only): +# +# - SIGINT, SIGTERM and SIGHUP are blocked for this thread with +# pthread_sigmask where it exists; +# - a handler that knows about sections (marked ``_deepctl_defers``, like +# speak's signal guard) defers them through ``defer_signal``; any other +# handler, including the default action, is swapped for a recorder for +# the duration, since another thread can take the signal and Python then +# runs the handler on the main thread; +# - when the outermost section ends, the most severe signal that arrived +# (SIGTERM, then SIGHUP, then SIGINT; the rest are dropped) is raised +# again with ``signal.raise_signal``, which never kills the process the +# way ``os.kill`` on our own pid does on Windows. +# +# A signal ignored with SIG_IGN, or held by a handler Python can't put +# back (``getsignal`` returns None), is left alone. + +_PRIORITY = tuple( + sig + for sig in ( + getattr(signal, "SIGTERM", None), + getattr(signal, "SIGHUP", None), + getattr(signal, "SIGINT", None), + ) + if sig is not None +) +_critical = 0 +_deferred: list[int] = [] + + +def _record(signum: int, _frame: object) -> None: + _deferred.append(signum) + + +_record._deepctl_defers = True # type: ignore[attr-defined] + + +def most_severe(signals: list[int]) -> int | None: + """The signal to act on from several: SIGTERM > SIGHUP > SIGINT.""" + for sig in _PRIORITY: + if sig in signals: + return int(sig) + return signals[0] if signals else None + + +def _set_mask(how: int) -> set[int] | None: + if not hasattr(signal, "pthread_sigmask"): + return None + try: + return set(signal.pthread_sigmask(how, _PRIORITY)) + except (OSError, ValueError): + return None + + +@contextmanager +def critical_section() -> Iterator[None]: + global _critical + main = threading.current_thread() is threading.main_thread() + # Count first: a pending handler that runs inside the calls below then + # defers instead of raising, so the cleanup in ``finally`` always runs. + _critical += 1 + blocked: set[int] | None = None + swapped: dict[int, Any] = {} + try: + if main and _critical == 1: + blocked = _set_mask(signal.SIG_BLOCK) + for sig in _PRIORITY: + current = signal.getsignal(sig) + if current is signal.SIG_IGN or current is None: + continue + if getattr(current, "_deepctl_defers", False): + continue + signal.signal(sig, _record) + swapped[sig] = current + yield + finally: + try: + restore_handlers(swapped) + finally: + try: + if blocked is not None: + # Restored while still inside the section, so a handler + # that knows about sections defers rather than raising + # past this. The call sets the mask, then runs pending + # handlers; one that raises still leaves the mask set. + signal.pthread_sigmask(signal.SIG_SETMASK, blocked) + finally: + _critical -= 1 + if not _critical and _deferred: + signum = most_severe(_deferred) + _deferred.clear() + if signum is not None and main: + signal.raise_signal(signum) + + +def restore_handlers(saved: dict[int, Any]) -> None: + """Put every saved handler back. ``signal.signal`` runs a pending + handler before it switches; if that handler raises, the switch didn't + happen, so it is retried, and the exception (that signal, delivered) + is raised once every handler is back.""" + caught: BaseException | None = None + for sig, previous in saved.items(): + for _ in range(3): + try: + signal.signal(sig, previous) + break + except BaseException as exc: + caught = caught or exc + if caught is not None: + raise caught + + +def defer_signal(signum: int) -> bool: + """For a signal handler: inside a critical section, remember the signal + (the most severe one is raised again when the section ends) and return + True.""" + if not _critical: + return False + _deferred.append(signum) + return True + + +def _is_special(mode: int) -> bool: + return stat.S_ISCHR(mode) or stat.S_ISFIFO(mode) + + +def _open_special(path: str, checked: os.stat_result) -> Target | Refusal: + """Open a device or FIFO now, before the request, and make sure the + opened file is the one that was checked. + + Non-blocking, so a FIFO with no reader fails with ENXIO instead of + hanging; blocking mode is restored for the writes. + """ + flags = ( + os.O_WRONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOCTTY", 0) + | getattr(os, "O_CLOEXEC", 0) + ) + try: + fd = os.open(path, flags) + except OSError as exc: + if exc.errno == errno.ENXIO and stat.S_ISFIFO(checked.st_mode): + # The OS text ("No such device or address") is misleading here. + exc = OSError(errno.ENXIO, _NO_READER, path) + return Refusal("unwritable", exc) + try: + opened = os.fstat(fd) + if not os.path.samestat(opened, checked) or not _is_special(opened.st_mode): + raise OSError(errno.EEXIST, _CHANGED, path) + os.set_blocking(fd, True) + except OSError as exc: + os.close(fd) + return Refusal("unwritable", exc) + return Target(path, "special", fd=fd) + + +def _parent_refusal(path: str) -> Refusal | None: + """A missing, non-directory or unwritable parent directory.""" + parent = os.path.dirname(os.path.abspath(path)) + try: + st = os.stat(parent) + except OSError as exc: + return Refusal("unwritable", exc) + if not stat.S_ISDIR(st.st_mode): + return Refusal("unwritable", _oserror(errno.ENOTDIR, parent)) + if not os.access(parent, os.W_OK | os.X_OK): + return Refusal("unwritable", _oserror(errno.EACCES, parent)) + return None + + +def plan_target(path: str, *, overwrite: bool) -> Target | Refusal: + """Decide, before the request, how ``path`` will be written. + + ``overwrite`` is ``--force`` for ``--out-file`` and always true for the + deprecated ``-o``, which keeps 0.4.0's overwrite. + """ + try: + followed: os.stat_result | None = os.stat(path) + except OSError: + followed = None + if followed is not None: + err = _stderr_stat() + if err is not None and os.path.samestat(followed, err): + # Audio would be mixed with dg's own messages. + return Refusal("unwritable", OSError(errno.EINVAL, _STDERR, path)) + out = stdout_stat() + if out is not None and os.path.samestat(followed, out): + return Target(path, "stdout", ident=(followed.st_dev, followed.st_ino)) + + try: + st = os.lstat(path) + except FileNotFoundError: + refusal = _parent_refusal(path) + return refusal or Target(path, "new") + except OSError as exc: + return Refusal("unwritable", exc) + + # What the path names: the link's target for a symlink. + named = followed if stat.S_ISLNK(st.st_mode) else st + if named is not None and stat.S_ISDIR(named.st_mode): + return Refusal("unwritable", _oserror(errno.EISDIR, path)) + if named is not None and _is_special(named.st_mode): + # /dev/null, a named pipe, or a link to one (/dev/stdout). + return _open_special(path, named) + if named is not None and not stat.S_ISREG(named.st_mode): + # A block device or a socket. + return Refusal("unwritable", OSError(errno.EINVAL, _NOT_SPECIAL, path)) + + # An existing regular file, or a symlink to one (or to nothing). + if not overwrite: + return Refusal("exists") + parent = os.path.dirname(os.path.abspath(path)) + if parent == "/dev/fd" or _FD_DIR.fullmatch(os.path.realpath(parent)): + return Refusal("unwritable", OSError(errno.EINVAL, _FD_PATH, path)) + # Permission bits only: never setuid, setgid or sticky. + keep = None if stat.S_ISLNK(st.st_mode) else stat.S_IMODE(st.st_mode) & 0o777 + return _parent_refusal(path) or Target(path, "replace", mode=keep) + + +class AudioFile: + """An audio file opened on the first chunk and removed on failure. + + ``write`` raises :class:`WriteFailed` (or :class:`ReaderGone` for EPIPE) + so a failed write can't be mistaken for a failed request. + """ + + def __init__(self, target: Target) -> None: + self.target = target + self.bytes_written = 0 + # A device or FIFO was opened (and checked) before the request. + self._fd: int | None = target.fd + self._tmp: str | None = None + self._created = False + self._done = False + + @property + def path(self) -> str: + return self.target.path + + @property + def is_stdout(self) -> bool: + return self.target.kind == "stdout" + + @property + def opened(self) -> bool: + return self._fd is not None or (self.is_stdout and self._created) + + def open(self) -> None: + """Create or open the file. Idempotent.""" + if self.opened: + return + kind, path = self.target.kind, self.target.path + try: + if kind == "new": + with critical_section(): + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + except FileExistsError as exc: + raise Exists(path) from exc + self._fd, self._created = fd, True + elif kind == "replace": + import tempfile + + directory, name = os.path.split(os.path.abspath(path)) + with critical_section(): + self._fd, self._tmp = tempfile.mkstemp( + dir=directory, prefix=f".{name}.", suffix=".part" + ) + elif kind == "special": + raise OSError(errno.EBADF, _CHANGED, path) # opened in plan + else: + self._check_stdout() + except (Exists, WriteFailed): + raise + except OSError as exc: + raise WriteFailed(exc, stdout=False) from exc + + def _check_stdout(self) -> None: + """stdout must still be the file that was checked. The audio + goes to fd 1 itself; the path is never opened again.""" + out = stdout_stat() + if out is None or (out.st_dev, out.st_ino) != self.target.ident: + # It changed since the check; don't write into it unasked. + raise Exists(self.target.path) + self._created = True + + def write(self, data: bytes) -> None: + """Write ``data`` in full (opening the file first if needed).""" + if not data: + return + self.open() + if self.is_stdout: + out = sys.stdout.buffer + try: + out.write(data) + out.flush() + except BrokenPipeError as exc: + raise ReaderGone from exc + except OSError as exc: + raise WriteFailed(exc, stdout=True) from exc + self.bytes_written += len(data) + return + assert self._fd is not None + view = memoryview(data) + try: + while view: + n = os.write(self._fd, view) + view = view[n:] + except BrokenPipeError as exc: + raise ReaderGone from exc + except OSError as exc: + raise WriteFailed(exc, stdout=False) from exc + self.bytes_written += len(data) + + def commit(self) -> None: + """Finish the file: close it, and put a temp file in place. Not + done until the rename succeeds; a failure removes what we made.""" + if self._done: + return + with critical_section(): + fd, self._fd = self._fd, None + try: + if fd is not None: + try: + if self._tmp is not None: + # An existing file keeps its permission bits + # ; a new file or a replaced symlink gets + # 0666 & ~umask, as open() would. + mode = self.target.mode + os.fchmod(fd, 0o666 & ~_UMASK if mode is None else mode) + finally: + os.close(fd) + if self._tmp is not None: + os.replace(self._tmp, self.target.path) + self._tmp = None + except OSError as exc: + self.abort() + raise WriteFailed(exc, stdout=False) from exc + self._done = True + + def abort(self) -> None: + """Close and remove whatever this run created. Never raises; safe + to call again, and a temp file is always removed.""" + if self._done: + return + with critical_section(): + fd, self._fd = self._fd, None + ours: os.stat_result | None = None + if fd is not None: + try: + ours = os.fstat(fd) + except OSError: + pass + try: + os.close(fd) + except OSError: + pass + if self._tmp is not None: + try: + os.unlink(self._tmp) + except OSError: + pass + self._tmp = None + elif self._created and self.target.kind == "new" and ours is not None: + # Only the file this run created: if the path now names + # another file (renamed or replaced meanwhile), leave it + # alone. + try: + if os.path.samestat(os.lstat(self.target.path), ours): + os.unlink(self.target.path) + except OSError: + pass + self._done = True diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py index 640d3bce..bc03c6e9 100644 --- a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py @@ -588,7 +588,7 @@ def test_handle_flux_rejects_non_raw_encoding( mock_sys.stdin.isatty.return_value = True mock_sys.stdout.isatty.return_value = True - with pytest.raises(click.ClickException, match="not supported for Flux"): + with pytest.raises(click.ClickException, match="Flux voices stream raw audio"): command.handle( config=mock_config, auth_manager=mock_auth_manager, @@ -1734,7 +1734,8 @@ def test_handle_play_rejects_mp3_on_pcm_only_player( assert result is not None assert result.status == "error" - assert "cannot decode mp3" in result.message + assert "can't decode mp3" in result.message + assert "--out-file FILE" in result.message mock_client.speak_text.assert_not_called() @@ -1819,7 +1820,8 @@ def test_check_playable_rejects_raw_audio(self): "ffplay", is_flux=False, encoding="linear16", container="none" ) assert aura_raw is not None - assert "no container" in aura_raw + assert "can't tell its format" in aura_raw + assert "--out-file FILE" in aura_raw def test_check_playable_aplay_rejects_every_compressed_encoding(self): for encoding in ("mp3", "aac", "opus", "flac"): @@ -1827,7 +1829,7 @@ def test_check_playable_aplay_rejects_every_compressed_encoding(self): "aplay", is_flux=False, encoding=encoding, container=None ) assert problem is not None - assert f"cannot decode {encoding}" in problem + assert f"can't decode {encoding}" in problem assert "linear16 (WAV)" in problem def test_check_playable_paplay_decodes_flac_and_ogg_but_not_mp3_or_aac(self): @@ -1848,7 +1850,7 @@ def test_check_playable_paplay_decodes_flac_and_ogg_but_not_mp3_or_aac(self): "paplay", is_flux=False, encoding=encoding, container=None ) assert problem is not None - assert f"'paplay' cannot decode {encoding}" in problem + assert f"'paplay' can't decode {encoding}" in problem assert "flac" in problem and "opus" in problem def test_check_playable_ffplay_and_afplay_accept_every_compressed_encoding(self): @@ -1936,3 +1938,15 @@ def test_speak_result_defaults(self): assert result.output_path == "" assert result.model == "" assert result.bytes_written == 0 + + +def test_help_names_the_file_options_value_file(): + """--out-file and -o/--output take a FILE, and --help says so.""" + from deepctl_cmd_speak.command import SpeakCommand + from deepctl_core import PluginManager + + cmd = PluginManager()._create_click_command(SpeakCommand()) + help_text = " ".join(cmd.get_help(click.Context(cmd)).split()) + assert "--out-file FILE Save the audio to FILE." in help_text + assert "-o, --output FILE Deprecated: use --out-file." in help_text + assert "--out-file TEXT" not in help_text diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py index 9d5087ba..a02c5ac9 100644 --- a/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_imports.py @@ -44,6 +44,7 @@ def _env(tmp_path: Path) -> dict[str, str]: "module", [ "deepctl_cmd_speak.command", + "deepctl_cmd_speak.sinks", "deepctl_cmd_speak.messages", "deepctl_cmd_speak.models", ], diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_output.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_output.py new file mode 100644 index 00000000..53e85044 --- /dev/null +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_output.py @@ -0,0 +1,1227 @@ +"""``--out-file``, ``--force`` and the deprecated ``-o``. + +The process-level checks (fake API request counts, a real ``/dev/stdout``) +run in a Docker harness; these cover the logic. +""" + +from __future__ import annotations + +import errno +import os +import shlex +import stat +import threading +import time +from pathlib import Path +from typing import Any +from unittest.mock import Mock, patch + +import pytest +from deepctl_cmd_speak import command as speak +from deepctl_cmd_speak import sinks +from deepctl_cmd_speak.command import SpeakCommand +from deepctl_cmd_speak.models import SpeakCodedError, SpeakResult +from deepctl_core import AuthManager, Config, DeepgramClient + + +class _Cut(Exception): + """Stands in for the SDK error a dropped connection raises.""" + + +def _stream(chunks: list[bytes], *, fail_after: bool = False, check=None): + for i, chunk in enumerate(chunks): + if check is not None: + check(i) + yield chunk + if fail_after: + raise _Cut("connection reset") + + +def _run(cmd: SpeakCommand, client: Mock, **kwargs: Any): + base: dict[str, Any] = { + "text": "Hello", + "output": None, + "out_file": None, + "force": False, + "model": "aura-2-asteria-en", + "encoding": None, + "container": None, + "sample_rate": None, + "file": None, + } + base.update(kwargs) + planned = cmd.pre_auth_check(Mock(spec=Config), **base) + if planned is not None: + return planned + return cmd.handle( + config=Mock(spec=Config), + auth_manager=Mock(spec=AuthManager), + client=client, + **base, + ) + + +@pytest.fixture(autouse=True) +def _env(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(speak, "_deprecation_shown", False) + monkeypatch.setattr(speak, "_agentic", False) + monkeypatch.setattr(speak, "is_ai_agent", lambda: False) + # stdout is "a pipe" unless a test says otherwise. + monkeypatch.setattr(sinks, "stdout_stat", lambda: None) + # The signal guard re-delivers a signal after _CLOSE_TIMEOUT_S if the + # run is still unwinding. A loaded test host must never get there by + # accident: a timer that fired after the guard restored SIG_DFL would + # end pytest itself. The one test of the timer sets its own value. + monkeypatch.setattr(speak, "_SIGNAL_GRACE_S", 600.0) + import signal + + getsignal = signal.getsignal # the real one, even if a test patches it + watched = [getattr(signal, n) for n in ("SIGINT", "SIGTERM", "SIGHUP")] + handlers = {sig: getsignal(sig) for sig in watched} + old = os.umask(0o022) + yield + os.umask(old) + # No test may leave a signal handler, a section or a deferred signal. + assert {sig: getsignal(sig) for sig in watched} == handlers + assert sinks._critical == 0 + assert not sinks._deferred + + +@pytest.fixture +def fake_sys(): + with patch("deepctl_cmd_speak.command.sys") as mock_sys: + mock_sys.stdin.isatty.return_value = True + mock_sys.stdout.isatty.return_value = False + mock_sys.argv = ["dg", "speak", "Hello"] + yield mock_sys + + +@pytest.fixture +def client(): + return Mock(spec=DeepgramClient) + + +# ── plan_target (before the request) ──────────────────────────────────── + + +class TestPlanTarget: + def test_missing_file_in_existing_dir_is_new(self, tmp_path): + assert sinks.plan_target(str(tmp_path / "a.wav"), overwrite=False) == ( + sinks.Target(str(tmp_path / "a.wav"), "new") + ) + + def test_existing_regular_file_is_refused_without_overwrite(self, tmp_path): + f = tmp_path / "a.wav" + f.write_bytes(b"old") + assert sinks.plan_target(str(f), overwrite=False) == sinks.Refusal("exists") + assert sinks.plan_target(str(f), overwrite=True).kind == "replace" + + @pytest.mark.parametrize("dangling", [False, True]) + def test_symlink_is_refused_without_overwrite(self, tmp_path, dangling): + link = tmp_path / "a.wav" + target = tmp_path / "elsewhere.wav" + if not dangling: + target.write_bytes(b"precious") + link.symlink_to(target) + assert sinks.plan_target(str(link), overwrite=False) == sinks.Refusal("exists") + assert sinks.plan_target(str(link), overwrite=True).kind == "replace" + + def test_directory_is_unwritable(self, tmp_path): + refusal = sinks.plan_target(str(tmp_path), overwrite=True) + assert isinstance(refusal, sinks.Refusal) + assert refusal.reason == "unwritable" + assert refusal.error.errno == errno.EISDIR + + def test_missing_parent_is_unwritable(self, tmp_path): + refusal = sinks.plan_target(str(tmp_path / "nope" / "a.wav"), overwrite=False) + assert refusal.reason == "unwritable" + assert refusal.error.errno == errno.ENOENT + + def test_parent_that_is_a_file_is_unwritable(self, tmp_path): + (tmp_path / "f").write_text("x") + refusal = sinks.plan_target(str(tmp_path / "f" / "a.wav"), overwrite=False) + assert refusal.reason == "unwritable" + + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores permissions") + def test_unwritable_parent_is_unwritable(self, tmp_path): + d = tmp_path / "ro" + d.mkdir() + d.chmod(0o555) + try: + refusal = sinks.plan_target(str(d / "a.wav"), overwrite=False) + assert refusal.reason == "unwritable" + assert refusal.error.errno == errno.EACCES + finally: + d.chmod(0o755) + + def test_character_device_is_special_and_opened_now(self): + target = sinks.plan_target(os.devnull, overwrite=False) + assert target.kind == "special" + assert target.fd is not None + assert os.get_blocking(target.fd) # O_NONBLOCK cleared for the writes + os.close(target.fd) + + def test_fifo_with_a_reader_is_special(self, tmp_path): + fifo = tmp_path / "f" + os.mkfifo(fifo) + reader = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) + try: + target = sinks.plan_target(str(fifo), overwrite=False) + assert target.kind == "special" + os.close(target.fd) + finally: + os.close(reader) + + def test_fifo_with_no_reader_is_refused_not_waited_on(self, tmp_path): + """An open() that would block until a reader comes is ENXIO.""" + fifo = tmp_path / "f" + os.mkfifo(fifo) + refusal = sinks.plan_target(str(fifo), overwrite=False) + assert refusal.reason == "unwritable" + assert refusal.error.errno == errno.ENXIO + assert refusal.error.strerror == "no program is reading from that pipe" + + def test_socket_is_refused(self, tmp_path): + """Only character devices and FIFOs are written as they are.""" + import socket + + path = tmp_path / "s" + sock = socket.socket(socket.AF_UNIX) + sock.bind(str(path)) + try: + refusal = sinks.plan_target(str(path), overwrite=True) + assert refusal.reason == "unwritable" + assert refusal.error.strerror == ( + "it isn't a file, a pipe or a device like /dev/null" + ) + finally: + sock.close() + + def test_symlink_to_a_device_is_special(self, tmp_path): + link = tmp_path / "a.wav" + link.symlink_to(os.devnull) + target = sinks.plan_target(str(link), overwrite=False) + assert target.kind == "special" + os.close(target.fd) + + def test_special_path_swapped_before_the_open_is_refused( + self, tmp_path, monkeypatch + ): + """The opened file must be the device that was checked.""" + victim = tmp_path / "victim" + victim.write_bytes(b"VICTIM") + link = tmp_path / "a.wav" + link.symlink_to(os.devnull) + real_open = os.open + + def swap_then_open(path, flags, *a): + if str(path) == str(link): + link.unlink() + link.symlink_to(victim) + return real_open(path, flags, *a) + + monkeypatch.setattr(sinks.os, "open", swap_then_open) + refusal = sinks.plan_target(str(link), overwrite=False) + assert refusal.reason == "unwritable" + assert refusal.error.strerror == "something replaced it before writing started" + assert victim.read_bytes() == b"VICTIM" + + def test_replace_keeps_the_old_mode(self, tmp_path): + f = tmp_path / "a.wav" + f.write_bytes(b"old") + f.chmod(0o600) + assert sinks.plan_target(str(f), overwrite=True).mode == 0o600 + + def test_replace_of_a_symlink_has_no_mode_to_keep(self, tmp_path): + (tmp_path / "t").write_bytes(b"x") + (tmp_path / "a.wav").symlink_to("t") + assert sinks.plan_target(str(tmp_path / "a.wav"), overwrite=True).mode is None + + def test_same_file_as_stdout_is_stdout(self, tmp_path, monkeypatch): + f = tmp_path / "out.wav" + f.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(f)) + # Even an existing regular file: it's stdout, not an overwrite. + assert sinks.plan_target(str(f), overwrite=False).kind == "stdout" + + +# ── AudioFile (at and after the first chunk) ──────────────────────────── + + +class TestAudioFile: + def test_new_file_appears_on_first_write_with_umask_mode(self, tmp_path): + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + assert not path.exists() + out.write(b"abc") + assert path.exists() + out.commit() + assert path.read_bytes() == b"abc" + assert stat.S_IMODE(path.stat().st_mode) == 0o644 + + def test_abort_removes_the_partial_file(self, tmp_path): + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + out.write(b"abc") + out.abort() + assert not path.exists() + + def test_o_excl_race_raises_exists_and_leaves_their_file(self, tmp_path): + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + path.write_bytes(b"theirs") # created between the check and the open + with pytest.raises(sinks.Exists): + out.write(b"abc") + out.abort() + assert path.read_bytes() == b"theirs" + + def test_replace_writes_a_0600_temp_then_replaces(self, tmp_path): + path = tmp_path / "a.wav" + path.write_bytes(b"old") + out = sinks.AudioFile(sinks.Target(str(path), "replace")) + out.write(b"new") + temps = [p for p in tmp_path.iterdir() if p.name.endswith(".part")] + assert len(temps) == 1 + assert stat.S_IMODE(temps[0].stat().st_mode) == 0o600 + assert path.read_bytes() == b"old" # untouched until commit + out.commit() + assert path.read_bytes() == b"new" + assert stat.S_IMODE(path.stat().st_mode) == 0o666 & ~sinks._UMASK + assert [p.name for p in tmp_path.iterdir()] == ["a.wav"] + + def test_replace_swaps_a_symlink_and_leaves_its_target(self, tmp_path): + target = tmp_path / "precious.txt" + target.write_bytes(b"precious") + link = tmp_path / "a.wav" + link.symlink_to(target) + out = sinks.AudioFile(sinks.Target(str(link), "replace")) + out.write(b"audio") + out.commit() + assert not link.is_symlink() + assert link.read_bytes() == b"audio" + assert target.read_bytes() == b"precious" + + def test_replace_keeps_a_0600_file_0600(self, tmp_path): + """Replacing doesn't widen an existing file's mode.""" + path = tmp_path / "a.wav" + path.write_bytes(b"old") + path.chmod(0o600) + out = sinks.AudioFile(sinks.plan_target(str(path), overwrite=True)) + out.write(b"new") + out.commit() + assert path.read_bytes() == b"new" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + def test_abort_leaves_a_file_that_replaced_ours(self, tmp_path): + """Only the file this run created is removed.""" + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + out.write(b"partial") + path.unlink() + path.write_bytes(b"theirs") + out.abort() + assert path.read_bytes() == b"theirs" + + def test_failed_replace_keeps_the_old_file(self, tmp_path): + path = tmp_path / "a.wav" + path.write_bytes(b"old") + out = sinks.AudioFile(sinks.Target(str(path), "replace")) + out.write(b"partial") + out.abort() + assert path.read_bytes() == b"old" + assert [p.name for p in tmp_path.iterdir()] == ["a.wav"] + + def test_write_error_is_write_failed(self, tmp_path, monkeypatch): + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + + def full(fd, data): + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(sinks.os, "write", full) + with pytest.raises(sinks.WriteFailed) as info: + out.write(b"abc") + assert info.value.error.errno == errno.ENOSPC + assert info.value.stdout is False + + def test_special_file_is_not_removed(self, tmp_path): + fifo_like = sinks.AudioFile(sinks.plan_target(os.devnull, overwrite=False)) + fifo_like.write(b"abc") + fifo_like.abort() + assert os.path.exists(os.devnull) + + def test_nothing_written_means_nothing_created(self, tmp_path): + path = tmp_path / "a.wav" + out = sinks.AudioFile(sinks.Target(str(path), "new")) + out.write(b"") + out.abort() + assert not path.exists() + + +# ── The command: checks before the request ────────────────────────────── + + +class TestPreRequest: + def test_existing_out_file_is_refused_with_no_request(self, fake_sys, client): + Path("a.wav").write_bytes(b"old") + cmd = SpeakCommand() + result = _run(cmd, client, out_file="a.wav") + assert isinstance(result, SpeakCodedError) + assert result.code == "invalid_argument" + assert result.error == "a.wav already exists; dg won't overwrite it." + assert result.hint == "Choose a new file name, or pass --force to overwrite." + assert result.next_step is None + client.speak_text.assert_not_called() + client.speak_text_stream.assert_not_called() + assert Path("a.wav").read_bytes() == b"old" + + def test_existing_symlink_is_refused(self, fake_sys, client): + Path("t").write_bytes(b"precious") + Path("a.wav").symlink_to("t") + result = _run(SpeakCommand(), client, out_file="a.wav") + assert result.code == "invalid_argument" + client.speak_text.assert_not_called() + + def test_force_replaces_the_file(self, fake_sys, client): + Path("a.wav").write_bytes(b"old") + client.speak_text.return_value = _stream([b"ne", b"w"]) + result = _run(SpeakCommand(), client, out_file="a.wav", force=True) + assert isinstance(result, SpeakResult) + assert result.output_path == "a.wav" + assert result.bytes_written == 3 + assert Path("a.wav").read_bytes() == b"new" + + def test_force_replaces_a_symlink_not_its_target(self, fake_sys, client): + Path("t").write_bytes(b"precious") + Path("a.wav").symlink_to("t") + client.speak_text.return_value = _stream([b"audio"]) + _run(SpeakCommand(), client, out_file="a.wav", force=True) + assert not Path("a.wav").is_symlink() + assert Path("a.wav").read_bytes() == b"audio" + assert Path("t").read_bytes() == b"precious" + + def test_missing_parent_is_output_unwritable_with_no_request( + self, fake_sys, client + ): + result = _run(SpeakCommand(), client, out_file="nope/a.wav") + assert isinstance(result, SpeakCodedError) + assert result.code == "output_unwritable" + assert result.next_step is None + assert result.details["path"] == "nope/a.wav" + assert result.details["reason"] == os.strerror(errno.ENOENT) + client.speak_text.assert_not_called() + + def test_directory_target_is_output_unwritable(self, fake_sys, client): + Path("d").mkdir() + result = _run(SpeakCommand(), client, out_file="d") + assert result.code == "output_unwritable" + assert result.details["reason"] == os.strerror(errno.EISDIR) + + def test_out_file_and_o_conflict(self, fake_sys, client, capsys): + cmd = SpeakCommand() + result = _run(cmd, client, out_file="a.wav", output="b.wav") + assert result.code == "invalid_argument" + assert result.error == ( + "Use either --out-file or -o/--output for the audio file, not both." + ) + assert result.next_step == "dg speak --help" + assert "DEPRECATED" not in capsys.readouterr().err + client.speak_text.assert_not_called() + + @pytest.mark.parametrize("value", ["json", "yaml", "table", "csv", "JSON"]) + def test_o_format_value_is_refused_with_a_runnable_next_step( + self, fake_sys, client, monkeypatch, value + ): + monkeypatch.setattr( + speak, + "_argv", + lambda: ["--profile", "work", "speak", "Q&A at 5", "-o", value], + ) + result = _run(SpeakCommand(), client, output=value, text="Q&A at 5") + assert result.code == "invalid_argument" + assert result.error == ( + f"dg speak -o is the audio file name, so -o {value} would write a " + f"file named {value}." + ) + ns = result.next_step + assert ns is not None + assert shlex.join(shlex.split(ns)) == ns + tokens = shlex.split(ns) + assert tokens[:6] == ["dg", "--profile", "work", "-o", value, "speak"] + assert tokens[6:8] == ["Q&A at 5", "--out-file"] + assert tokens[8].startswith("speech-") and tokens[8].endswith(".mp3") + assert len(tokens) == 9 + client.speak_text.assert_not_called() + + def test_o_text_points_at_out_file(self, fake_sys, client, monkeypatch): + """There's no root text format: the fix drops -o and adds --out-file.""" + monkeypatch.setattr(speak, "_argv", lambda: ["-q", "speak", "hi", "-o", "text"]) + result = _run(SpeakCommand(), client, output="text", text="hi") + assert result.code == "invalid_argument" + assert result.error == ( + "dg speak -o text would write the audio to a file named text." + ) + assert result.hint == ( + "Pass --out-file FILE to save the audio. There's no text output format." + ) + tokens = shlex.split(result.next_step) + assert tokens[:5] == ["dg", "-q", "speak", "hi", "--out-file"] + assert tokens[5].startswith("speech-") and len(tokens) == 6 + client.speak_text.assert_not_called() + + def test_o_text_next_step_is_null_with_a_key_on_argv( + self, fake_sys, client, monkeypatch + ): + monkeypatch.setattr( + speak, "_argv", lambda: ["-k", "sk_sentinel", "speak", "hi", "-o", "text"] + ) + result = _run(SpeakCommand(), client, output="text", text="hi") + assert result.error.startswith("dg speak -o text") + assert result.next_step is None + + def test_o_format_next_step_is_null_with_a_key_on_argv( + self, fake_sys, client, monkeypatch + ): + monkeypatch.setattr( + speak, + "_argv", + lambda: ["--api-key", "sk_sentinel", "speak", "hi", "-o", "json"], + ) + result = _run(SpeakCommand(), client, output="json", text="hi") + assert result.next_step is None + assert "sk_sentinel" not in result.model_dump_json() + + def test_o_file_is_deprecated_once_and_overwrites( + self, fake_sys, client, capsys, monkeypatch + ): + Path("a.wav").write_bytes(b"old") + client.speak_text.side_effect = lambda **_: _stream([b"new"]) + cmd = SpeakCommand() + _run(cmd, client, output="a.wav") + _run(cmd, client, output="a.wav") + err = capsys.readouterr().err + assert err.count("Deprecated:") == 1 + assert ( + "dg speak -o/--output FILE will set the output format in 1.0. Use " + "--out-file FILE instead (add --force to overwrite, as -o does)." in err + ) + assert Path("a.wav").read_bytes() == b"new" + + def test_deprecation_prints_under_quiet_in_agent_grammar( + self, fake_sys, client, capsys, monkeypatch + ): + from deepctl_core import output + + monkeypatch.setattr(speak, "_agentic", True) + monkeypatch.setitem(output._output_config, "quiet", True) + client.speak_text.return_value = _stream([b"new"]) + _run(SpeakCommand(), client, output="a.wav") + lines = [ + ln for ln in capsys.readouterr().err.splitlines() if "DEPRECATED" in ln + ] + assert lines == [ + "DEPRECATED: dg speak -o/--output FILE will set the output format " + "in 1.0. Use --out-file FILE instead (add --force to overwrite, as " + "-o does)." + ] + + @pytest.mark.parametrize("fmt", ["json", "yaml"]) + def test_f1_with_explicit_json_is_refused(self, fake_sys, client, monkeypatch, fmt): + f = Path("out.wav") + f.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(f)) + monkeypatch.setattr(speak, "get_output_format", lambda: fmt) + monkeypatch.setattr(speak, "format_source", lambda: "explicit") + result = _run(SpeakCommand(), client, out_file="out.wav") + assert result.code == "invalid_argument" + name = fmt.upper() + assert result.error == ( + "dg speak would write the audio to stdout, so there's no room for " + f"the {name} result." + ) + assert result.hint == ( + f"Give --out-file a different file from stdout, or don't ask for " + f"{name} output." + ) + assert result.next_step == "dg speak --help" + client.speak_text.assert_not_called() + + def test_f1_with_implicit_json_writes_audio_only( + self, fake_sys, client, monkeypatch, capsys + ): + f = Path("out.wav") + f.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(f)) + monkeypatch.setattr(speak, "get_output_format", lambda: "json") + monkeypatch.setattr(speak, "format_source", lambda: "hard_agent") + monkeypatch.setattr(speak, "_agentic", True) + written: list[bytes] = [] + stdout = Mock() + stdout.write.side_effect = written.append + monkeypatch.setattr(sinks.sys, "stdout", Mock(buffer=stdout)) + client.speak_text.return_value = _stream([b"RIFF", b"data"]) + + cmd = SpeakCommand() + result = _run(cmd, client, out_file="out.wav") + + assert result is None # no document after the audio + assert cmd.stdout_is_protocol is True + assert written == [b"RIFF", b"data"] + assert "OK: Wrote 8 bytes to stdout" in capsys.readouterr().err + + +# ── The command: no file on any error ─────────────────────────────────── + + +class TestNoPartialFile: + def test_request_error_leaves_no_file(self, fake_sys, client): + def unauthorized(**_): + raise RuntimeError("status_code: 401") + yield b"" # pragma: no cover + + client.speak_text.side_effect = lambda **kw: unauthorized(**kw) + with pytest.raises(Exception, match="401"): + _run(SpeakCommand(), client, out_file="a.wav") + assert not Path("a.wav").exists() + + def test_cut_stream_leaves_no_partial_file(self, fake_sys, client): + client.speak_text.return_value = _stream([b"a", b"b"], fail_after=True) + with pytest.raises(Exception, match="connection reset"): + _run(SpeakCommand(), client, out_file="a.wav") + assert list(Path().iterdir()) == [] + + def test_cut_stream_keeps_the_old_o_file(self, fake_sys, client): + Path("a.wav").write_bytes(b"old bytes") + client.speak_text.return_value = _stream([b"a"], fail_after=True) + with pytest.raises(Exception, match="connection reset"): + _run(SpeakCommand(), client, output="a.wav") + assert Path("a.wav").read_bytes() == b"old bytes" + assert [p.name for p in Path().iterdir()] == ["a.wav"] + + def test_zero_chunks_leave_no_file(self, fake_sys, client): + client.speak_text.return_value = _stream([]) + result = _run(SpeakCommand(), client, out_file="a.wav") + assert result.status == "error" + assert not Path("a.wav").exists() + + def test_file_write_error_is_output_unwritable_with_no_partial_file( + self, fake_sys, client, monkeypatch + ): + real_write = os.write + calls = {"n": 0} + + def fills_up(fd, data): + calls["n"] += 1 + if calls["n"] > 1: + raise OSError(errno.ENOSPC, "No space left on device") + return real_write(fd, data) + + monkeypatch.setattr(sinks.os, "write", fills_up) + client.speak_text.return_value = _stream([b"a", b"b", b"c"]) + cmd = SpeakCommand() + + result = _run(cmd, client, out_file="a.wav") + + assert isinstance(result, SpeakCodedError) + assert result.code == "output_unwritable" + assert result.error == ( + "Couldn't write the output to a.wav: No space left on device." + ) + assert result.details == {"path": "a.wav", "reason": "No space left on device"} + assert cmd.exit_code_for(result) == 1 + assert not Path("a.wav").exists() + + def test_flux_file_is_created_on_the_first_frame(self, fake_sys, client): + seen: list[bool] = [] + + def check(i): + if i == 1: + seen.append(Path("a.wav").exists()) + + client.speak_text_stream.return_value = _stream( + [b"\x01\x00", b"\x02\x00"], check=check + ) + result = _run(SpeakCommand(), client, out_file="a.wav", model="flux-alexis-en") + assert seen == [True] + assert isinstance(result, SpeakResult) + assert Path("a.wav").read_bytes()[:4] == b"RIFF" + + def test_flux_cut_stream_leaves_no_file(self, fake_sys, client): + client.speak_text_stream.return_value = _stream([b"\x01\x00"], fail_after=True) + with pytest.raises(Exception, match="connection reset"): + _run(SpeakCommand(), client, out_file="a.wav", model="flux-alexis-en") + assert list(Path().iterdir()) == [] + + def test_machine_format_result_keeps_the_040_keys(self, fake_sys, client): + client.speak_text.return_value = _stream([b"abc"]) + result = _run(SpeakCommand(), client, out_file="a.wav") + assert list(result.model_dump()) == [ + "status", + "message", + "output_path", + "model", + "bytes_written", + "played", + ] + + +# ── SIGTERM / SIGHUP mid-stream ───────────────────────────────────────── + + +class TestTerminateMidStream: + @pytest.fixture + def died(self, monkeypatch): + calls: list[int] = [] + + def fake_die(signum): + calls.append(signum) + raise SystemExit(128 + signum) + + monkeypatch.setattr(speak, "_die_by_signal", fake_die) + return calls + + def _signalling_stream(self, sig, chunks): + import signal as _signal + + def gen(): + for i, chunk in enumerate(chunks): + yield chunk + if i == 0: + os.kill(os.getpid(), sig) + raise AssertionError("the signal should have stopped the stream") + + return gen(), _signal + + @pytest.mark.parametrize("signame", ["SIGTERM", "SIGHUP"]) + @pytest.mark.parametrize("model", ["aura-2-asteria-en", "flux-alexis-en"]) + @pytest.mark.parametrize("force", [False, True]) + def test_cleanup_runs_then_the_signal_ends_the_process( + self, fake_sys, client, died, signame, model, force + ): + import signal + + sig = getattr(signal, signame) + if force: + Path("a.wav").write_bytes(b"OLD") + stream, _ = self._signalling_stream( + sig, [b"\x01\x00", b"\x02\x00", b"\x03\x00"] + ) + if model.startswith("flux"): + client.speak_text_stream.return_value = stream + else: + client.speak_text.return_value = stream + before = signal.getsignal(sig) + + with pytest.raises(SystemExit) as info: + _run(SpeakCommand(), client, out_file="a.wav", force=force, model=model) + + assert info.value.code == 128 + sig + assert died == [sig] + if force: + assert Path("a.wav").read_bytes() == b"OLD" + assert [p.name for p in Path().iterdir()] == ["a.wav"] + else: + assert list(Path().iterdir()) == [] + assert signal.getsignal(sig) == before # handler restored + + def test_a_stalled_unwind_is_cut_short_by_the_second_signal( + self, died, monkeypatch + ): + """If unwinding hangs (an unanswered close handshake), the + re-delivered signal runs the cleanup and ends the process.""" + import signal + + monkeypatch.setattr(speak, "_SIGNAL_GRACE_S", 0.05) + cleaned: list[bool] = [] + with ( + pytest.raises(SystemExit), + speak._terminate_cleanly(lambda: cleaned.append(True)), + ): + try: + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) # unreachable: the handler raises + except speak._Terminated: + # "Stuck" unwinding: wait until the re-delivered signal ends + # it (the handler raises SystemExit through this wait). + threading.Event().wait(60) + assert cleaned == [True] + assert died == [signal.SIGTERM] + + def test_second_ctrl_c_cleans_up_and_cancels_like_the_first(self, monkeypatch): + """N-3: the second Ctrl-C (or the timer) unwinds by KeyboardInterrupt + again, so the usual cancel path, exit 2 and exit handlers follow; + the hard-exit backstop is cancelled with the guard.""" + import signal + + exits: list[int] = [] + monkeypatch.setattr(speak.os, "_exit", exits.append) + cleaned: list[bool] = [] + with ( + pytest.raises(KeyboardInterrupt), + speak._terminate_cleanly(lambda: cleaned.append(True)), + ): + try: + os.kill(os.getpid(), signal.SIGINT) + except KeyboardInterrupt: + os.kill(os.getpid(), signal.SIGINT) + assert cleaned == [True] + assert exits == [] + timers = [t for t in threading.enumerate() if isinstance(t, threading.Timer)] + assert all(t.finished.is_set() for t in timers) # cancelled + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + + +# ── Round 3: ignored signals, special bits, critical sections, fd paths ── + + +class TestIgnoredSignalsStayIgnored: + """nohup, background jobs and `trap ''` keep their dispositions.""" + + @pytest.mark.parametrize("signame", ["SIGHUP", "SIGTERM", "SIGINT"]) + def test_sig_ign_is_left_alone(self, signame): + import signal + + sig = getattr(signal, signame) + before = signal.signal(sig, signal.SIG_IGN) + try: + with speak._terminate_cleanly(lambda: None): + assert signal.getsignal(sig) is signal.SIG_IGN + os.kill(os.getpid(), sig) # ignored: nothing raises + assert signal.getsignal(sig) is signal.SIG_IGN + finally: + signal.signal(sig, before) + + def test_a_handler_python_cant_restore_is_not_replaced(self, monkeypatch): + import signal + + installed: list[int] = [] + real_signal = signal.signal + real_getsignal = signal.getsignal + monkeypatch.setattr( + speak.signal, + "getsignal", + lambda sig: None if sig == signal.SIGHUP else real_getsignal(sig), + ) + + def recording(sig, handler): + installed.append(sig) + return real_signal(sig, handler) + + monkeypatch.setattr(speak.signal, "signal", recording) + with speak._terminate_cleanly(lambda: None): + pass + assert signal.SIGHUP not in installed + + @pytest.mark.parametrize("model", ["aura-2-asteria-en", "flux-alexis-en"]) + @pytest.mark.parametrize("out_file", [True, False]) + def test_the_stream_completes_through_an_ignored_sighup( + self, fake_sys, client, monkeypatch, model, out_file + ): + import signal + + written: list[bytes] = [] + fake_sys.stdout.buffer = Mock(write=written.append) + + def gen(): + yield b"\x01\x00" + os.kill(os.getpid(), signal.SIGHUP) + yield b"\x02\x00" + + if model.startswith("flux"): + client.speak_text_stream.return_value = gen() + else: + client.speak_text.return_value = gen() + before = signal.signal(signal.SIGHUP, signal.SIG_IGN) + try: + kw = {"out_file": "a.wav"} if out_file else {} + result = _run(SpeakCommand(), client, model=model, **kw) + finally: + signal.signal(signal.SIGHUP, before) + if out_file: + assert isinstance(result, SpeakResult) + assert Path("a.wav").exists() + else: + assert result is None + assert b"\x02\x00" in written + + +class TestRound3Sinks: + def test_replace_drops_setuid_setgid_and_sticky(self, tmp_path): + path = tmp_path / "a.wav" + path.write_bytes(b"old") + path.chmod(0o4755) + out = sinks.AudioFile(sinks.plan_target(str(path), overwrite=True)) + out.write(b"new") + out.commit() + assert stat.S_IMODE(path.stat().st_mode) == 0o755 + + def test_a_signal_inside_the_rename_leaves_no_temp_file( + self, tmp_path, monkeypatch + ): + """N-1 (the reviewer's repro): os.replace interrupted; abort still + removes the temp file.""" + path = tmp_path / "a.wav" + path.write_bytes(b"old") + out = sinks.AudioFile(sinks.plan_target(str(path), overwrite=True)) + out.write(b"new") + + def boom(*a): + raise speak._Terminated(15) + + monkeypatch.setattr(sinks.os, "replace", boom) + with pytest.raises(speak._Terminated): + out.commit() + out.abort() + assert [p.name for p in tmp_path.iterdir()] == ["a.wav"] + assert path.read_bytes() == b"old" + + def test_signals_wait_for_the_end_of_a_critical_section(self): + import signal + + seen: list[str] = [] + + with ( + pytest.raises(speak._Terminated), + speak._terminate_cleanly(lambda: None), + ): + with sinks.critical_section(): + os.kill(os.getpid(), signal.SIGTERM) + seen.append("still inside") + seen.append("unreachable") + assert seen == ["still inside"] + + def test_stdout_target_is_checked_on_fd_1_without_reopening( + self, tmp_path, monkeypatch + ): + f = tmp_path / "out.wav" + f.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(f)) + target = sinks.plan_target(str(f), overwrite=False) + assert target.kind == "stdout" + opened: list[str] = [] + real_open = os.open + monkeypatch.setattr( + sinks.os, "open", lambda p, *a: opened.append(p) or real_open(p, *a) + ) + other = tmp_path / "other" + other.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(other)) + with pytest.raises(sinks.Exists): + sinks.AudioFile(target).open() + assert opened == [] + + @pytest.mark.skipif(not os.path.isdir("/dev/fd"), reason="no /dev/fd") + def test_fd_path_with_force_is_refused_before_the_request(self, tmp_path): + victim = tmp_path / "v" + victim.write_bytes(b"VICTIM") + fd = os.open(victim, os.O_WRONLY | os.O_APPEND) + try: + path = f"/dev/fd/{fd}" + refusal = sinks.plan_target(path, overwrite=True) + assert refusal.reason == "unwritable" + assert refusal.error.strerror == ( + "--force can't replace a file descriptor path" + ) + assert sinks.plan_target(path, overwrite=False) == sinks.Refusal("exists") + finally: + os.close(fd) + assert victim.read_bytes() == b"VICTIM" + + def test_stderr_is_refused(self, tmp_path, monkeypatch): + f = tmp_path / "err.log" + f.write_bytes(b"") + monkeypatch.setattr(sinks, "_stderr_stat", lambda: os.stat(f)) + refusal = sinks.plan_target(str(f), overwrite=False) + assert refusal.reason == "unwritable" + assert refusal.error.strerror == ( + "it's stderr, which carries this command's messages" + ) + + +class TestSignalGuardEdges: + """Windows re-delivery, the restore window, masks left blocked, bursts, and + cleanup outside the guard.""" + + def test_signal_inside_importlib_comes_again_later(self, monkeypatch): + """A raise inside importlib's lock bookkeeping can leave the import + lock held (a hang), so the guard lets the signal come again.""" + import signal + + from deepctl_core import signals + + later: list[int] = [] + inside = [True] + monkeypatch.setattr(signals, "in_importlib", lambda frame: inside[0]) + monkeypatch.setattr(signals, "deliver_later", later.append) + signals._deferred_since.clear() + cleaned: list[bool] = [] + with ( + pytest.raises(speak._Terminated), + speak._terminate_cleanly(lambda: cleaned.append(True)), + ): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(0.05) # handled, deferred: no raise + assert later == [signal.SIGTERM] + inside[0] = False + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) # unreachable: the handler raises + assert cleaned == [] # first signal: unwinding, no forced cleanup + signals._deferred_since.clear() + + @staticmethod + def _importlib_frame(): + import importlib + import sys + + seen = [] + + class Finder: + def find_spec(self, name, path=None, target=None): + if name == "deepctl_speak_probe_xyz": + seen.append(sys._getframe(1)) + + finder = Finder() + sys.meta_path.insert(0, finder) + try: + with pytest.raises(ModuleNotFoundError): + importlib.import_module("deepctl_speak_probe_xyz") + finally: + sys.meta_path.remove(finder) + return seen[0] + + def test_guard_handler_defers_over_an_importlib_frame(self, monkeypatch): + """The guard's own handler holds a signal back inside importlib.""" + import signal + + from deepctl_core import signals + + later: list[int] = [] + monkeypatch.setattr(signals, "deliver_later", later.append) + signals._deferred_since.clear() + frame = self._importlib_frame() + with speak._terminate_cleanly(lambda: None): + installed = signal.getsignal(signal.SIGTERM) + installed(signal.SIGTERM, frame) # no raise: deferred + assert later == [signal.SIGTERM] + signals._deferred_since.clear() + + def test_guard_handler_coalesces_a_nested_signal(self, monkeypatch): + """The guard's handler uses core's coalescing, so a signal that + lands while it runs isn't judged by the handler's own frame.""" + import signal + import sys + + from deepctl_core import signals + + calls: list[int] = [] + + def nested_then_defer(signum, frame): + calls.append(signum) + if len(calls) == 1: + signal.getsignal(signal.SIGTERM)(signum, sys._getframe()) + return True + + monkeypatch.setattr(speak, "_defer_in_importlib", nested_then_defer) + with speak._terminate_cleanly(lambda: None): + signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None) + assert calls == [signal.SIGTERM] # the nested call didn't run the body + assert signals._in_handler == [] + + def test_windows_never_kills_its_own_process(self, monkeypatch): + import signal + + own = os.getpid() + killed: list[tuple[int, int]] = [] + monkeypatch.setattr(sinks.os, "kill", lambda p, s: killed.append((p, s))) + interrupted: list[object] = [] + import _thread + + monkeypatch.setattr(_thread, "interrupt_main", lambda *a: interrupted.append(a)) + monkeypatch.setattr(speak.os, "name", "nt") + speak._send_to_main(signal.SIGTERM) + assert interrupted == [(signal.SIGTERM,)] + with pytest.raises(SystemExit) as info: + speak._end_by_signal(signal.SIGTERM) + assert info.value.code == 128 + signal.SIGTERM + assert not [k for k in killed if k[0] == own] + + def test_deferred_signal_is_raised_not_killed(self, monkeypatch): + import signal + + def no_kill(pid, sig): + raise AssertionError("os.kill on our own pid") + + monkeypatch.setattr(sinks.os, "kill", no_kill) + seen: list[int] = [] + + def h(signum, frame): + if sinks.defer_signal(signum): + return + seen.append(signum) + + h._deepctl_defers = True + before = signal.signal(signal.SIGHUP, h) + try: + with sinks.critical_section(): + sinks.defer_signal(signal.SIGHUP) + assert seen == [signal.SIGHUP] + finally: + signal.signal(signal.SIGHUP, before) + + def test_most_severe_deferred_signal_wins(self): + """N-1: TERM then INT during a section ends by TERM.""" + import signal + + seen: list[int] = [] + + def h(signum, frame): + if sinks.defer_signal(signum): + return + seen.append(signum) + + h._deepctl_defers = True + saved = {s: signal.signal(s, h) for s in (signal.SIGINT, signal.SIGTERM)} + try: + with sinks.critical_section(): + sinks.defer_signal(signal.SIGTERM) + sinks.defer_signal(signal.SIGINT) + assert seen == [signal.SIGTERM] + finally: + for s, old in saved.items(): + signal.signal(s, old) + + @pytest.mark.parametrize("signame", ["SIGINT", "SIGTERM"]) + def test_a_signal_during_the_restore_reaches_the_restored_handler( + self, monkeypatch, signame + ): + """S-2: closed guard, original handlers back, no live timer, no + cleanup; the signal goes to the handler that was there before.""" + import signal + + sig = getattr(signal, signame) + outer: list[int] = [] + saved = { + s: signal.signal(s, lambda n, f: outer.append(n)) + for s in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) + } + mine = {s: signal.getsignal(s) for s in saved} + real = signal.signal + armed = [True] + + def wrapped(s, h): + if armed[0] and getattr(h, "__name__", "") != "handler": + armed[0] = False + os.kill(os.getpid(), sig) # arrives during the restore + return real(s, h) + + cleaned: list[bool] = [] + try: + monkeypatch.setattr(speak.signal, "signal", wrapped) + with speak._terminate_cleanly(lambda: cleaned.append(True)): + pass + monkeypatch.setattr(speak.signal, "signal", real) + + def settled() -> bool: + timers = [ + t for t in threading.enumerate() if isinstance(t, threading.Timer) + ] + return bool(outer) and all(t.finished.is_set() for t in timers) + + # Wait for the signal and any timer, up to a generous deadline. + deadline = time.monotonic() + 5.0 + while not settled() and time.monotonic() < deadline: + time.sleep(0.01) + assert outer == [sig] + assert cleaned == [] + assert {s: signal.getsignal(s) for s in saved} == mine + live = [t for t in threading.enumerate() if isinstance(t, threading.Timer)] + assert all(t.finished.is_set() for t in live) + finally: + for s, old in saved.items(): + real(s, old) + + @pytest.mark.skipif( + not hasattr(__import__("signal"), "pthread_sigmask"), reason="POSIX" + ) + def test_mask_is_restored_under_a_signal_flood(self): + """S-3: the mask is always restored, even when a pending handler + raises inside the section's own setup. Run in a child process: the + flood can land on any bytecode.""" + import subprocess + import sys as _sys + + code = """ +import os, signal, threading, time +from deepctl_cmd_speak import sinks +class Boom(BaseException): pass +stop = threading.Event() +def h(signum, frame): + if sinks.defer_signal(signum) or stop.is_set(): + return + raise Boom +h._deepctl_defers = True +signal.signal(signal.SIGTERM, h) +go = threading.Event() +def flood(): + go.wait() + while not stop.is_set(): + os.kill(os.getpid(), signal.SIGTERM); time.sleep(0.0002) +threading.Thread(target=flood, daemon=True).start() +stuck = raises = 0 +go.set() +end = time.monotonic() + 1.5 +while True: + try: + while time.monotonic() < end: + try: + with sinks.critical_section(): + pass + except Boom: + raises += 1 + if signal.SIGTERM in signal.pthread_sigmask(signal.SIG_BLOCK, set()): + stuck += 1 + signal.pthread_sigmask(signal.SIG_SETMASK, set()) + stop.set() + break + except Boom: + raises += 1 +time.sleep(0.05) +print(f"raises={raises} masks_left_blocked={stuck} critical={sinks._critical}") +""" + p = subprocess.run( + [_sys.executable, "-c", code], capture_output=True, text=True, timeout=60 + ) + assert p.returncode == 0, p.stderr[-2000:] + assert "masks_left_blocked=0 critical=0" in p.stdout, p.stdout + assert "raises=0" not in p.stdout + + def test_abort_outside_the_guard_still_removes_the_temp_file(self, tmp_path): + """N-2: a Ctrl-C taken by another thread while abort() runs with no + guard installed is held until the removal is done.""" + import signal + + path = tmp_path / "a.wav" + path.write_bytes(b"OLD") + out = sinks.AudioFile(sinks.plan_target(str(path), overwrite=True)) + out.write(b"x" * 100) + real_close = os.close + fd = out._fd + + def close_then_signal(n): + real_close(n) + if n == fd: + t = threading.Thread(target=os.kill, args=(os.getpid(), signal.SIGINT)) + t.start() + t.join() + time.sleep(0.05) + + sinks.os.close = close_then_signal + try: + with pytest.raises(KeyboardInterrupt): + out.abort() # delivered after the section, not inside it + finally: + sinks.os.close = real_close + assert [p.name for p in tmp_path.iterdir()] == ["a.wav"] + assert path.read_bytes() == b"OLD" + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler diff --git a/packages/deepctl-core/src/deepctl_core/base_group_command.py b/packages/deepctl-core/src/deepctl_core/base_group_command.py index 43a5d197..1c64e565 100644 --- a/packages/deepctl-core/src/deepctl_core/base_group_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_group_command.py @@ -166,6 +166,7 @@ def get_click_group(self) -> click.Group: required=arg.get("required", False), is_flag=arg.get("is_flag", False), multiple=arg.get("multiple", False), + **({"metavar": arg["metavar"]} if "metavar" in arg else {}), )(group) else: group = click.argument( diff --git a/packages/deepctl-core/src/deepctl_core/plugin_manager.py b/packages/deepctl-core/src/deepctl_core/plugin_manager.py index 894ac79a..59b4da93 100644 --- a/packages/deepctl-core/src/deepctl_core/plugin_manager.py +++ b/packages/deepctl-core/src/deepctl_core/plugin_manager.py @@ -1212,6 +1212,8 @@ def _add_command_arguments( } if not is_flag: option_kwargs["type"] = arg.get("type", str) + if "metavar" in arg: + option_kwargs["metavar"] = arg["metavar"] cmd = click.option(*arg.get("names", []), **option_kwargs)(cmd) else: # Add as argument diff --git a/packages/deepctl-core/src/deepctl_core/signals.py b/packages/deepctl-core/src/deepctl_core/signals.py new file mode 100644 index 00000000..3716d07f --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/signals.py @@ -0,0 +1,207 @@ +"""Ending the process by a signal, for commands that catch one to clean up. + +Stdlib only and cheap to import: command modules use it at import time. +""" + +from __future__ import annotations + +import _imp +import _thread +import os +import signal +import sys +import threading +import time +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + from types import FrameType + + Handler = Callable[[int, FrameType | None], None] + +__all__ = [ + "coalescing", + "defer_in_importlib", + "deliver_later", + "die_by_signal", + "in_importlib", + "install_interrupt_handler", + "send_to_main", +] + + +def die_by_signal(signum: int) -> None: + """End the process by ``signum`` with its default action, so the wait + status is "killed by signal N" (shells report 128+N). + + stdout and stderr are flushed first. PID 1 in a container ignores a + default-action signal, so if the process is still here afterwards it + exits with the shell's 128+N. POSIX only: on Windows ``os.kill`` of our + own pid is TerminateProcess, so callers there exit instead. + """ + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except Exception: + pass + try: + signal.signal(signum, signal.SIG_DFL) + if hasattr(signal, "pthread_sigmask"): + # A signal left blocked would stay pending and never kill. + signal.pthread_sigmask(signal.SIG_UNBLOCK, {signum}) + os.kill(os.getpid(), signum) + time.sleep(0.05) + except Exception: + pass + os._exit(128 + signum) + + +#: Where importlib's own lock bookkeeping runs. A KeyboardInterrupt raised +#: in one of these frames can land between acquiring the global import lock +#: and the ``try`` that releases it (and in the module-lock weakref callback +#: it is swallowed outright), so every later import, and the run, hangs. +_IMPORTLIB_FILES = frozenset( + {"", ""} +) + +#: The longest a SIGINT is postponed, from its first deferral. +_MAX_DEFER_S = 0.2 + +#: A deferral older than this was never delivered (the handler changed, or +#: the re-send was lost): the next signal starts a fresh window. +_STALE_DEFER_S = 1.0 + +_deferred_since: list[float] = [] +_in_handler: list[bool] = [] + + +def _own_file() -> str: + return _own_file.__code__.co_filename + + +def in_importlib(frame: FrameType | None) -> bool: + """True when the interrupted ``frame`` is importlib's own code. Frames of + this module are skipped: a second signal can arrive while a handler here + runs, and what matters is where the first one interrupted.""" + while frame is not None and frame.f_code.co_filename == _own_file(): + frame = frame.f_back + return frame is not None and frame.f_code.co_filename in _IMPORTLIB_FILES + + +def send_to_main(signum: int) -> None: + """Deliver ``signum`` to the main thread (from any thread). + + POSIX: ``signal.pthread_kill`` to the main thread, a real signal, so it + also interrupts a blocking wait there. ``_thread.interrupt_main`` would + only set a flag that is checked once that wait returns. + Windows: ``_thread.interrupt_main``; ``os.kill`` on our own pid there is + TerminateProcess, which would skip the cleanup. + """ + if os.name != "nt" and hasattr(signal, "pthread_kill"): + main = threading.main_thread().ident + if main is not None: + signal.pthread_kill(main, signum) + return + try: + _thread.interrupt_main(signal.Signals(signum)) + except (ValueError, TypeError): + _thread.interrupt_main() # a signal it can't simulate: Ctrl-C + + +def _resend(signum: int) -> None: + # Under load, starting this timer can take longer than its delay. Wait + # for the handler that started it to return, or the signal would arrive + # inside that handler and be taken for a second Ctrl-C. + deadline = time.monotonic() + 1.0 + while _in_handler and time.monotonic() < deadline: + time.sleep(0.001) + send_to_main(signum) + + +def deliver_later(signum: int, delay: float = 0.01) -> None: + """Send ``signum`` to the main thread again after ``delay`` seconds.""" + timer = threading.Timer(delay, _resend, (signum,)) + timer.daemon = True + timer.start() + + +def defer_in_importlib(signum: int, frame: FrameType | None) -> bool: + """For a signal handler about to raise: if ``frame`` is importlib's own + code, send the signal again a moment later and return True (the handler + returns without raising). After ``_MAX_DEFER_S`` of deferrals (unless + the import lock is held right now), or outside importlib, return False + so the handler raises as usual.""" + now = time.monotonic() + if _deferred_since and now - _deferred_since[0] > _STALE_DEFER_S: + _deferred_since.clear() + if in_importlib(frame) and ( + not _deferred_since + or now - _deferred_since[0] < _MAX_DEFER_S + or _import_lock_held() + ): + try: + deliver_later(signum) + except RuntimeError: + # No new thread at interpreter shutdown: deliver it now instead. + _deferred_since.clear() + return False + if not _deferred_since: + _deferred_since.append(now) + return True + _deferred_since.clear() + return False + + +def _import_lock_held() -> bool: + """Past the cap, a signal is still held back while the import lock is + held: in importlib's frames that means inside its locked bookkeeping, the + one place a raise is unsafe. (A main thread waiting for another thread's + lock is blocked in C, where no handler runs.)""" + try: + return bool(_imp.lock_held()) + except Exception: + return False + + +def coalescing(handler: Handler) -> Handler: + """Wrap a signal handler that uses :func:`defer_in_importlib`. + + A signal that arrives while the handler runs is coalesced into the one + being handled (its frame would be the handler's, not where the first + signal landed), and a re-send waits for the handler to return. + """ + + def wrapped(signum: int, frame: FrameType | None) -> None: + if _in_handler: + return # a second signal while the first is handled + _in_handler.append(True) + try: + handler(signum, frame) + finally: + _in_handler.clear() + + wrapped.__wrapped__ = handler # type: ignore[attr-defined] + return wrapped + + +def _interrupt(signum: int, frame: FrameType | None) -> None: + if defer_in_importlib(signum, frame): + return + raise KeyboardInterrupt + + +_on_interrupt = coalescing(_interrupt) + + +def install_interrupt_handler() -> None: + """Replace Python's default SIGINT handler with one that raises + ``KeyboardInterrupt`` the same way, except inside importlib's own code + (see :func:`defer_in_importlib`), where the signal comes again a moment + later. A SIGINT that is ignored or handled elsewhere is left alone. + """ + try: + if signal.getsignal(signal.SIGINT) is signal.default_int_handler: + signal.signal(signal.SIGINT, _on_interrupt) + except (OSError, ValueError): + pass # not the main thread, or no SIGINT here diff --git a/packages/deepctl-core/tests/unit/test_base_group.py b/packages/deepctl-core/tests/unit/test_base_group.py index 93529724..eed79d2e 100644 --- a/packages/deepctl-core/tests/unit/test_base_group.py +++ b/packages/deepctl-core/tests/unit/test_base_group.py @@ -239,6 +239,23 @@ def handle_group( assert len(click_group.params) == 1 assert click_group.params[0].name == "verbose" + @pytest.mark.unit + def test_group_option_metavar_is_passed_through(self): + class GroupWithFile(BaseGroupCommand): + name = "testgroup" + help = "Test group" + + def get_arguments(self) -> list[dict[str, Any]]: + return [ + {"names": ["--out-file"], "is_option": True, "metavar": "FILE"} + ] + + def handle_group(self, *args: Any, **kwargs: Any) -> Any: + return None + + click_group = GroupWithFile().get_click_group() + assert "--out-file FILE" in click_group.get_help(click.Context(click_group)) + @pytest.mark.unit def test_group_callback_execution(self, mock_group_command_class): """Test that the group callback properly executes.""" diff --git a/packages/deepctl-core/tests/unit/test_plugin_manager.py b/packages/deepctl-core/tests/unit/test_plugin_manager.py index 180979b4..d18b3511 100644 --- a/packages/deepctl-core/tests/unit/test_plugin_manager.py +++ b/packages/deepctl-core/tests/unit/test_plugin_manager.py @@ -154,6 +154,28 @@ def test_function_name_conversion_for_hyphens( # Check that function name has underscores assert click_command.callback.__name__ == "test_hyphenated_command" + @pytest.mark.unit + def test_option_metavar_is_passed_through(self, plugin_manager): + """An argument dict's "metavar" names the value in --help.""" + + class Saves(BaseCommand): + name = "saves" + help = "Saves" + + def get_arguments(self) -> List[Dict[str, Any]]: + return [ + {"names": ["--out-file"], "is_option": True, "metavar": "FILE"}, + {"names": ["--note"], "is_option": True}, + ] + + def handle(self, *args: Any, **kwargs: Any) -> Any: + return None + + cmd = plugin_manager._create_click_command(Saves()) + help_text = cmd.get_help(click.Context(cmd)) + assert "--out-file FILE" in help_text + assert "--note TEXT" in help_text + @pytest.mark.unit def test_group_function_name_conversion_for_hyphens( self, plugin_manager, mock_group_command_class diff --git a/packages/deepctl-core/tests/unit/test_signals.py b/packages/deepctl-core/tests/unit/test_signals.py new file mode 100644 index 00000000..51ef7a5d --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_signals.py @@ -0,0 +1,255 @@ +"""deepctl_core.signals: the SIGINT handler that waits out importlib's own +lock bookkeeping, and the helpers behind it.""" + +from __future__ import annotations + +import _imp +import importlib +import os +import signal +import sys +import threading +import time + +import pytest +from deepctl_core import signals + + +@pytest.fixture(autouse=True) +def _restore_sigint(): + before = signal.getsignal(signal.SIGINT) + for state in (signals._deferred_since, signals._in_handler): + state.clear() + yield + signal.signal(signal.SIGINT, before) + for state in (signals._deferred_since, signals._in_handler): + state.clear() + + +def _importlib_frame(): + """A real frame of importlib's own code: the caller of a finder.""" + seen = [] + + class Finder: + def find_spec(self, name, path=None, target=None): + if name == "deepctl_signals_probe_xyz": + seen.append(sys._getframe(1)) + return None + + finder = Finder() + sys.meta_path.insert(0, finder) + try: + with pytest.raises(ModuleNotFoundError): + importlib.import_module("deepctl_signals_probe_xyz") + finally: + sys.meta_path.remove(finder) + return seen[0] + + +def test_replaces_only_the_default_handler(): + signal.signal(signal.SIGINT, signal.default_int_handler) + signals.install_interrupt_handler() + assert signal.getsignal(signal.SIGINT) is signals._on_interrupt + + signal.signal(signal.SIGINT, signal.SIG_IGN) + signals.install_interrupt_handler() + assert signal.getsignal(signal.SIGINT) is signal.SIG_IGN + + +def test_in_importlib_knows_importlibs_frames(): + assert signals.in_importlib(_importlib_frame()) is True + assert signals.in_importlib(sys._getframe()) is False + assert signals.in_importlib(None) is False + + +def test_a_signal_inside_this_modules_frames_is_judged_by_their_caller(): + """A second signal that lands in a handler's frames here counts as + landing where the first one interrupted: importlib, in this case.""" + ns: dict = {} + src = ( + "import sys\n" + "def find_spec(name, path=None, target=None):\n" + " if name == 'deepctl_signals_probe_own':\n" + " seen.append(sys._getframe(0))\n" + ) + ns["seen"] = seen = [] + exec(compile(src, signals._own_file(), "exec"), ns) + + class Finder: + find_spec = staticmethod(ns["find_spec"]) + + finder = Finder() + sys.meta_path.insert(0, finder) + try: + with pytest.raises(ModuleNotFoundError): + importlib.import_module("deepctl_signals_probe_own") + finally: + sys.meta_path.remove(finder) + own = seen[0] + assert own.f_code.co_filename == signals._own_file() + assert signals.in_importlib(own) is True + assert signals.in_importlib(sys._getframe()) is False + + +def test_a_second_ctrl_c_while_the_handler_runs_is_coalesced(monkeypatch): + later: list[int] = [] + monkeypatch.setattr(signals, "deliver_later", later.append) + + def nested_then_raise(signum, frame): + signals._on_interrupt(signum, frame) # nested: no raise, nothing sent + return False + + monkeypatch.setattr(signals, "defer_in_importlib", nested_then_raise) + with pytest.raises(KeyboardInterrupt): + signals._on_interrupt(signal.SIGINT, sys._getframe()) + assert later == [] + + +def test_the_resend_waits_for_the_handler_to_return(monkeypatch): + """Under load the re-send timer can fire before the handler that + started it returns; it waits, so the signal isn't taken as nested.""" + sent: list[int] = [] + monkeypatch.setattr(signals, "send_to_main", sent.append) + signals._in_handler.append(True) + t = threading.Thread(target=signals._resend, args=(signal.SIGINT,)) + t.start() + time.sleep(0.1) + assert sent == [] + signals._in_handler.clear() + t.join(2) + assert sent == [signal.SIGINT] + + +def test_raises_outside_importlib(): + with pytest.raises(KeyboardInterrupt): + signals._on_interrupt(signal.SIGINT, sys._getframe()) + + +def test_defers_inside_importlib(monkeypatch): + later: list[int] = [] + monkeypatch.setattr(signals, "deliver_later", later.append) + signals._on_interrupt(signal.SIGINT, _importlib_frame()) # no raise + assert later == [signal.SIGINT] + + +def test_deferral_is_capped(monkeypatch): + """After _MAX_DEFER_S of deferrals the handler raises anyway.""" + later: list[int] = [] + clock = [100.0] + monkeypatch.setattr(signals, "deliver_later", later.append) + monkeypatch.setattr(signals.time, "monotonic", lambda: clock[0]) + frame = _importlib_frame() + signals._on_interrupt(signal.SIGINT, frame) + clock[0] += signals._MAX_DEFER_S / 2 + signals._on_interrupt(signal.SIGINT, frame) + clock[0] += signals._MAX_DEFER_S + with pytest.raises(KeyboardInterrupt): + signals._on_interrupt(signal.SIGINT, frame) + assert len(later) == 2 + # The next Ctrl-C starts a fresh window. + signals._on_interrupt(signal.SIGINT, frame) + assert len(later) == 3 + + +def test_past_the_cap_a_held_import_lock_still_defers(monkeypatch): + later: list[int] = [] + clock = [100.0] + monkeypatch.setattr(signals, "deliver_later", later.append) + monkeypatch.setattr(signals.time, "monotonic", lambda: clock[0]) + frame = _importlib_frame() + signals._on_interrupt(signal.SIGINT, frame) + clock[0] += signals._MAX_DEFER_S + 0.05 # past the cap, not stale + _imp.acquire_lock() + try: + signals._on_interrupt(signal.SIGINT, frame) # lock held: no raise + finally: + _imp.release_lock() + assert len(later) == 2 + with pytest.raises(KeyboardInterrupt): + signals._on_interrupt(signal.SIGINT, frame) # lock free: raise + + +def test_a_stale_deferral_starts_a_fresh_window(monkeypatch): + """A deferral whose re-send never came (the handler changed meanwhile) + must not make a later Ctrl-C in importlib skip the deferral.""" + later: list[int] = [] + clock = [100.0] + monkeypatch.setattr(signals, "deliver_later", later.append) + monkeypatch.setattr(signals.time, "monotonic", lambda: clock[0]) + frame = _importlib_frame() + signals._on_interrupt(signal.SIGINT, frame) # deferred; re-send "lost" + clock[0] += signals._STALE_DEFER_S + 1 + signals._on_interrupt(signal.SIGINT, frame) # deferred again, no raise + assert later == [signal.SIGINT, signal.SIGINT] + + +def test_no_thread_for_the_resend_delivers_now(monkeypatch): + """At interpreter shutdown no thread can start: raise KeyboardInterrupt + rather than that RuntimeError.""" + + def no_thread(signum): + raise RuntimeError("can't create new thread at interpreter shutdown") + + monkeypatch.setattr(signals, "deliver_later", no_thread) + with pytest.raises(KeyboardInterrupt): + signals._on_interrupt(signal.SIGINT, _importlib_frame()) + assert signals._deferred_since == [] + + +def test_coalescing_releases_after_the_handler_raises(): + calls: list[int] = [] + + @signals.coalescing + def handler(signum, frame): + calls.append(signum) + handler(signum, frame) # nested: coalesced, inner body not run + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + handler(signal.SIGINT, None) + assert calls == [signal.SIGINT] + assert signals._in_handler == [] + + +@pytest.mark.skipif(not hasattr(signal, "pthread_kill"), reason="POSIX only") +def test_deliver_later_reaches_the_main_thread(): + got: list[int] = [] + before = signal.signal(signal.SIGUSR1, lambda s, f: got.append(s)) + try: + signals.deliver_later(signal.SIGUSR1, 0) + deadline = time.monotonic() + 2 + while not got and time.monotonic() < deadline: + time.sleep(0.005) + finally: + signal.signal(signal.SIGUSR1, before) + assert got == [signal.SIGUSR1] + + +@pytest.mark.skipif(not hasattr(signal, "pthread_kill"), reason="POSIX only") +def test_another_thread_holding_the_import_lock_doesnt_hold_up_ctrl_c(): + """Only the main thread's own importlib frames defer.""" + signal.signal(signal.SIGINT, signal.default_int_handler) + signals.install_interrupt_handler() + held, release = threading.Event(), threading.Event() + + def holder(): + _imp.acquire_lock() + try: + held.set() + release.wait(10) + finally: + _imp.release_lock() + + t = threading.Thread(target=holder, daemon=True) + t.start() + held.wait(5) + try: + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + os.kill(os.getpid(), signal.SIGINT) + while time.monotonic() - t0 < 2: + time.sleep(0.001) + assert time.monotonic() - t0 < 1.5 + finally: + release.set() + t.join(5) diff --git a/src/deepctl/_manifest.d/speak.json b/src/deepctl/_manifest.d/speak.json index ee9b0e99..e6469999 100644 --- a/src/deepctl/_manifest.d/speak.json +++ b/src/deepctl/_manifest.d/speak.json @@ -8,7 +8,7 @@ "requires_auth": true, "help": { "name": "speak", - "help": "Convert text to speech using Deepgram TTS\n\nExamples:\n dg speak \"Hello from Deepgram\" --play\n dg speak --list-voices\n dg speak \"Hello world\"\n dg speak \"Hello world\" -o hello.wav\n dg speak \"Hello world\" -o hello.wav --play\n dg speak --file message.txt -o output.wav\n dg speak \"Hello\" | ffplay -loglevel error -nodisp -autoexit -\n dg speak \"A little slower, please\" --speed 0.9 -o slow.wav\n dg speak \"So exciting!\" --expressivity 2 -o lively.wav # beta; default: 0\n dg speak \"Hello\" -m aura-2-asteria-en -o hello.mp3\n dg speak \"Hello\" -m aura-2-luna-en -o hello.wav --encoding linear16 --container wav\n echo \"Hello\" | dg speak -o hello.mp3 -m aura-2-asteria-en\n dg speak \"Hola, mundo\" -m aura-2-selena-es -o hola.mp3\n\n\nFor agents, use --agent-friendly.", + "help": "Convert text to speech using Deepgram TTS\n\nExamples:\n dg speak \"Hello from Deepgram\" --play\n dg speak --list-voices\n dg speak \"Hello world\"\n dg speak \"Hello world\" --out-file hello.wav\n dg speak \"Hello world\" --out-file hello.wav --play\n dg speak --file message.txt --out-file output.wav\n dg speak \"Hello\" | ffplay -loglevel error -nodisp -autoexit -\n dg speak \"A little slower, please\" --speed 0.9 --out-file slow.wav\n dg speak \"So exciting!\" --expressivity 2 --out-file lively.wav # beta; default: 0\n dg speak \"Hello\" -m aura-2-asteria-en --out-file hello.mp3\n dg speak \"Hello\" -m aura-2-luna-en --out-file hello.wav --encoding linear16 --container wav\n echo \"Hello\" | dg speak --out-file hello.mp3 -m aura-2-asteria-en\n dg speak \"Hola, mundo\" -m aura-2-selena-es --out-file hola.mp3\n\n\nFor agents, use --agent-friendly.", "short_help": "Text-to-speech", "epilog": null, "hidden": false, @@ -56,7 +56,7 @@ "is_flag": true, "count": false, "hidden": false, - "help": "Play the audio through your system player (ffplay, afplay, paplay, or aplay). Can be combined with -o to save and play." + "help": "Play the audio through your system player (ffplay, afplay, paplay, or aplay). Can be combined with --out-file to save and play." }, { "class": "option", @@ -220,14 +220,56 @@ }, "required": false, "nargs": 1, + "metavar": "FILE", + "eager": false, + "expose": true, + "multiple": false, + "is_flag": false, + "count": false, + "hidden": false, + "help": "Deprecated: use --out-file. Saves the audio to FILE and overwrites it. In 1.0, -o will set the output format." + }, + { + "class": "option", + "name": "force", + "opts": [ + "--force" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, "metavar": null, "eager": false, "expose": true, "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "With --out-file, overwrite FILE if it already exists." + }, + { + "class": "option", + "name": "out_file", + "opts": [ + "--out-file" + ], + "secondary_opts": [], + "type": { + "name": "text" + }, + "required": false, + "nargs": 1, + "metavar": "FILE", + "eager": false, + "expose": true, + "multiple": false, "is_flag": false, "count": false, "hidden": false, - "help": "Output file path (required if stdout is a terminal)" + "help": "Save the audio to FILE. An existing file isn't overwritten unless you add --force." }, { "class": "argument", From 05249042540e70a96c568ff6418d8ef5f18f334c Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:48:56 -0400 Subject: [PATCH 10/18] fix(speak): don't read stdin or write audio to a pipe in agent mode Under a hard AI-agent signal, dg speak with no text argument and no --file read stdin until it closed, which can be never, and audio bound for a pipe was written into whatever an agent's tool call was capturing. With a hard signal, stdin is now read only when the text is - ; anything else fails at once with E-SPEAK-NO-TEXT. Audio that would go to a stdout that isn't a terminal, a regular file or /dev/null, with no output file, --play or --list-voices, fails with E-SPEAK-NO-OUTPUT. Its next_step reruns the same command with --out-file and a new file name, and is null (with the matching hint) when the text came from stdin or the command line holds a secret. Both refusals come before the request. - as the text reads stdin in every mode. CI and people keep 0.4.0's behavior otherwise. The speak tests' person fixture lives in the root conftest, scoped to speak's tests: a second tests/unit/conftest.py would clash with dg mcp's when CI collects both. --- conftest.py | 13 + .../src/deepctl_cmd_speak/command.py | 59 +++- .../src/deepctl_cmd_speak/messages.py | 17 + .../src/deepctl_cmd_speak/sinks.py | 16 + .../tests/unit/test_speak_agent.py | 301 ++++++++++++++++++ .../tests/unit/test_speak_strings.py | 62 ++++ src/deepctl/_manifest.d/speak.json | 2 +- 7 files changed, 463 insertions(+), 7 deletions(-) create mode 100644 packages/deepctl-cmd-speak/tests/unit/test_speak_agent.py create mode 100644 packages/deepctl-cmd-speak/tests/unit/test_speak_strings.py diff --git a/conftest.py b/conftest.py index c94232e4..22734911 100644 --- a/conftest.py +++ b/conftest.py @@ -55,6 +55,19 @@ def _clean_deepgram_env(monkeypatch): monkeypatch.delenv(key, raising=False) +@pytest.fixture(autouse=True) +def _speak_tests_run_as_a_person(request, monkeypatch): + """Speak tests run as a person by default: the agent rules key off hard + agent markers, which a developer's own shell may set (CLAUDECODE=1). + Tests that need agent mode patch ``is_ai_agent`` themselves.""" + if "deepctl-cmd-speak" not in request.path.parts: + return + from deepctl_core.output import AI_AGENT_ENV_MARKERS + + for name in (*AI_AGENT_ENV_MARKERS, "OR_APP_NAME", "OR_SITE_URL"): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture(autouse=True) def _disable_telemetry(monkeypatch): """Keep tests that call `main()` from initializing real telemetry. diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index 29005ff2..be8da794 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -845,14 +845,17 @@ class SpeakCommand(BaseCommand): # containerized formats like mp3. 'dg speak "Hello" -m aura-2-asteria-en --out-file hello.mp3', 'dg speak "Hello" -m aura-2-luna-en --out-file hello.wav --encoding linear16 --container wav', - 'echo "Hello" | dg speak --out-file hello.mp3 -m aura-2-asteria-en', + 'echo "Hello" | dg speak - --out-file hello.mp3 -m aura-2-asteria-en', # Aura-2 Spanish voice (run `dg models` for the full list). 'dg speak "Hola, mundo" -m aura-2-selena-es --out-file hola.mp3', ] agent_help = ( "Convert text to speech using Deepgram's TTS API. " - "Text can be provided as an argument, from a file, or piped via stdin. " - "Audio is written to a file (--out-file; add --force to replace an " + "Text can be provided as an argument, from a file (--file), or from " + "stdin with - as the text (in agent mode, - is the only way to read " + "stdin). In agent mode, audio isn't written into a pipe: use " + "--out-file, --play, or redirect stdout to a file. " + "Audio is written to a file (--out-file; add --force to overwrite an " "existing file) or stdout for piping. -o/--output FILE is deprecated. " "By default (flux-* models, e.g. flux-alexis-en) uses Flux TTS / Speak " "v2, streaming over WebSocket and emitting raw audio; linear16 output is " @@ -870,7 +873,7 @@ def get_arguments(self) -> list[dict[str, Any]]: return [ { "name": "text", - "help": "Text to convert to speech", + "help": "Text to convert to speech, or - to read it from stdin", "required": False, "default": None, }, @@ -1026,11 +1029,24 @@ def _preflight(self, kwargs: dict[str, Any]) -> _Plan | SpeakCodedError: hint_agent=msg.H_SPEAK_O_FORMAT.format(prog=prog, value=legacy), next_step=self._o_format_next_step(prog, legacy, kwargs), ) + agent = is_ai_agent() + text = kwargs.get("text") + if agent and not text and not kwargs.get("file"): + # Never block on a stdin that may never close; "-" opts in. + return _coded( + msg.E_SPEAK_NO_TEXT.format(prog=prog), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_SPEAK_NO_TEXT, + hint_agent=msg.H_SPEAK_NO_TEXT, + next_step=_next_step([prog, self.name, "--help"]), + ) if legacy: _deprecate_output() path = out_file or legacy if not path: + if agent and not kwargs.get("play") and not sinks.stdout_takes_audio(): + return self._no_output(prog, kwargs) return _Plan() planned = sinks.plan_target( path, overwrite=bool(legacy) or bool(kwargs.get("force")) @@ -1063,6 +1079,34 @@ def _preflight(self, kwargs: dict[str, Any]) -> _Plan | SpeakCodedError: ) return _Plan(target=planned) + def _no_output(self, prog: str, kwargs: dict[str, Any]) -> SpeakCodedError: + """Under a hard agent signal, audio bound for a pipe is refused; + ``next_step`` reruns the command with a new ``--out-file``.""" + model = kwargs.get("model") or _DEFAULT_MODEL + out_file = _proposed_out_file( + is_flux=model.lower().startswith("flux-"), + encoding=kwargs.get("encoding"), + container=kwargs.get("container"), + ) + # Text read from stdin can't be sent again by a rerun. + next_step = ( + None + if kwargs.get("text") == "-" + else _next_step([prog, *_argv(), "--out-file", out_file]) + ) + hint = ( + msg.A_SPEAK_NO_OUTPUT.format(out_file=out_file) + if next_step is not None + else msg.A_SPEAK_NO_OUTPUT_NULL.format(out_file=out_file) + ) + return _coded( + msg.E_SPEAK_NO_OUTPUT.format(prog=prog), + code=msg.CODE_INVALID_ARGUMENT, + hint_human=msg.H_SPEAK_NO_OUTPUT, + hint_agent=hint, + next_step=next_step, + ) + def _o_format_next_step( self, prog: str, value: str, kwargs: dict[str, Any] ) -> str | None: @@ -1126,8 +1170,11 @@ def handle( if list_voices: return self._list_voices(client) - # Resolve text input: arg > --file > stdin - if not text and file_path: + # Resolve text input: arg > --file > stdin. "-" reads stdin in every + # mode (and is the only way to under a hard agent signal). + if text == "-": + text = sys.stdin.read().strip() + elif not text and file_path: path = Path(file_path) if not path.exists(): return _fail(f"File not found: {file_path}") diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py index e732402d..0e0313fa 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/messages.py @@ -34,6 +34,23 @@ "Give --out-file a different file from stdout, or don't ask for {format} output." ) +# invalid_argument: a hard agent signal and audio bound for a pipe. +E_SPEAK_NO_OUTPUT = "{prog} speak won't write audio into this pipe in agent mode." +H_SPEAK_NO_OUTPUT = "Pass --out-file FILE to save the audio." +A_SPEAK_NO_OUTPUT = "next_step saves the audio to {out_file}. Give the user that path." +# next_step is null: the text came from stdin, or the command line can't be +# repeated (it holds a key or a URL with credentials, or can't be quoted). +A_SPEAK_NO_OUTPUT_NULL = ( + "Run the same command again with --out-file {out_file} added. If the text " + "came from stdin, send it again the same way." +) + +# invalid_argument: a hard agent signal and no text argument. +E_SPEAK_NO_TEXT = ( + "{prog} speak needs the text as an argument, or - to read it from stdin." +) +H_SPEAK_NO_TEXT = "Pass the text in quotes. To read it from stdin, pass - explicitly." + # invalid_argument: --out-file names an existing file and --force is absent. E_OUT_FILE_EXISTS = "{out_file} already exists; {prog} won't overwrite it." H_OUT_FILE_EXISTS = "Choose a new file name, or pass --force to overwrite." diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py index 9440c99a..a3f7ac41 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/sinks.py @@ -106,6 +106,22 @@ def stdout_stat() -> os.stat_result | None: return None +def stdout_takes_audio() -> bool: + """True when stdout is a terminal, a regular file or the null device: + the destinations where agent mode still writes audio.""" + out = stdout_stat() + if out is None: + return False + if stat.S_ISREG(out.st_mode): + return True + try: + if os.path.samestat(out, os.stat(os.devnull)): + return True + return os.isatty(sys.stdout.fileno()) + except Exception: + return False + + def _oserror(code: int, path: str) -> OSError: return OSError(code, os.strerror(code), path) diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_agent.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_agent.py new file mode 100644 index 00000000..ea1e7eb8 --- /dev/null +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_agent.py @@ -0,0 +1,301 @@ +"""Agent rules for ``dg speak`` under a hard signal, and +the ``next_step`` rows for every speak coded error (``test_next_step``). +""" + +from __future__ import annotations + +import os +import re +import shlex +from typing import Any +from unittest.mock import Mock, patch + +import pytest +from deepctl_cmd_speak import command as speak +from deepctl_cmd_speak import sinks +from deepctl_cmd_speak.command import SpeakCommand +from deepctl_cmd_speak.models import SpeakCodedError +from deepctl_core import AuthManager, Config, DeepgramClient + +# The strict surface-0.5.0 schema's next_step grammar. +NEXT_STEP = re.compile( + r"^(?!sudo )(?:[A-Za-z0-9_@%+:,./-][A-Za-z0-9_@%+=:,./-]*|'[^']*'(?:\"'\"'[^']*')*)" + r"(?: (?:[A-Za-z0-9_@%+:,./-][A-Za-z0-9_@%+=:,./-]*|'[^']*'(?:\"'\"'[^']*')*))*$" +) +_REAL_TAKES_AUDIO = sinks.stdout_takes_audio +OUT_FILE = re.compile(r"^speech-\d{8}T\d{6}Z-[0-9a-f]{4}\.(mp3|wav|ogg|flac|aac|raw)$") + + +def _kwargs(**kw: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "text": "hi", + "output": None, + "out_file": None, + "force": False, + "model": "aura-2-asteria-en", + "encoding": None, + "container": None, + "sample_rate": None, + "file": None, + "play": False, + "list_voices": False, + } + base.update(kw) + return base + + +def _pre(argv: list[str], monkeypatch, **kw: Any): + monkeypatch.setattr(speak, "_argv", lambda: list(argv)) + return SpeakCommand().pre_auth_check(Mock(spec=Config), **_kwargs(**kw)) + + +def assert_e10(ns: str | None) -> None: + if ns is None: + return + assert shlex.join(shlex.split(ns)) == ns + assert NEXT_STEP.match(ns), ns + tokens = shlex.split(ns) + assert tokens[0] == "dg" + assert not any(re.fullmatch(r"<[A-Z_]+>", t) for t in tokens) + + +@pytest.fixture(autouse=True) +def _env(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(speak, "_deprecation_shown", False) + monkeypatch.setattr(speak, "_agentic", True) + monkeypatch.setattr(speak, "is_ai_agent", lambda: True) + monkeypatch.setattr(sinks, "stdout_stat", lambda: None) # a pipe + monkeypatch.setattr(sinks, "stdout_takes_audio", lambda: False) + + +# ── No text → never read stdin ───────────────────────────────────────── + + +class TestNoText: + @pytest.mark.parametrize("extra", [{}, {"out_file": "a.wav"}, {"text": ""}]) + def test_refused_without_reading_stdin(self, monkeypatch, extra): + with patch("deepctl_cmd_speak.command.sys") as mock_sys: + kw = {"text": None, **extra} + result = _pre(["speak"], monkeypatch, **kw) + mock_sys.stdin.read.assert_not_called() + assert isinstance(result, SpeakCodedError) + assert result.code == "invalid_argument" + assert result.error == ( + "dg speak needs the text as an argument, or - to read it from stdin." + ) + assert result.hint == ( + "Pass the text in quotes. To read it from stdin, pass - explicitly." + ) + assert result.next_step == "dg speak --help" + + def test_file_is_text(self, monkeypatch, tmp_path): + assert ( + _pre(["speak"], monkeypatch, text=None, file="t.txt", out_file="a.wav") + is None + ) + + def test_dash_reads_stdin_in_agent_mode(self, monkeypatch): + monkeypatch.setattr(sinks, "stdout_takes_audio", lambda: True) + assert _pre(["speak", "-"], monkeypatch, text="-") is None + cmd = SpeakCommand() + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = iter([b"a"]) + with patch("deepctl_cmd_speak.command.sys") as mock_sys: + mock_sys.stdin.read.return_value = "from stdin\n" + mock_sys.stdout.isatty.return_value = False + cmd.handle( + Mock(spec=Config), Mock(spec=AuthManager), client, **_kwargs(text="-") + ) + assert client.speak_text.call_args.kwargs["text"] == "from stdin" + + def test_dash_reads_stdin_for_a_person_too(self, monkeypatch): + monkeypatch.setattr(speak, "is_ai_agent", lambda: False) + cmd = SpeakCommand() + client = Mock(spec=DeepgramClient) + client.speak_text.return_value = iter([b"a"]) + with patch("deepctl_cmd_speak.command.sys") as mock_sys: + mock_sys.stdin.isatty.return_value = True + mock_sys.stdin.read.return_value = "typed" + mock_sys.stdout.isatty.return_value = False + cmd.handle( + Mock(spec=Config), Mock(spec=AuthManager), client, **_kwargs(text="-") + ) + assert client.speak_text.call_args.kwargs["text"] == "typed" + + def test_ci_only_keeps_reading_piped_stdin(self, monkeypatch): + monkeypatch.setattr(speak, "is_ai_agent", lambda: False) + assert _pre(["speak"], monkeypatch, text=None) is None + + +# ── Audio into a pipe ─────────────────────────────────────────────────── + + +class TestNoOutput: + def test_refused_with_a_rerun_that_saves_the_file(self, monkeypatch): + result = _pre(["speak", "hi", "-m", "aura-2-asteria-en"], monkeypatch) + assert isinstance(result, SpeakCodedError) + assert result.code == "invalid_argument" + assert ( + result.error == "dg speak won't write audio into this pipe in agent mode." + ) + tokens = shlex.split(result.next_step) + assert tokens[:6] == [ + "dg", + "speak", + "hi", + "-m", + "aura-2-asteria-en", + "--out-file", + ] + out_file = tokens[6] + assert OUT_FILE.match(out_file) and out_file.endswith(".mp3") + assert result.hint == ( + f"next_step saves the audio to {out_file}. Give the user that path." + ) + assert not os.path.exists(out_file) + assert_e10(result.next_step) + + def test_flux_default_proposes_a_wav(self, monkeypatch): + result = _pre(["speak", "hi"], monkeypatch, model=None) + assert shlex.split(result.next_step)[-1].endswith(".wav") + + def test_root_options_are_kept_verbatim(self, monkeypatch): + result = _pre(["--profile", "work", "speak", "hi"], monkeypatch) + assert shlex.split(result.next_step)[:4] == ["dg", "--profile", "work", "speak"] + + def test_text_from_stdin_has_no_next_step(self, monkeypatch): + result = _pre(["speak", "-"], monkeypatch, text="-") + assert result.next_step is None + out_file = result.hint.split("--out-file ")[1].split(" ")[0] + assert OUT_FILE.match(out_file) + assert result.hint == ( + f"Run the same command again with --out-file {out_file} added. If " + "the text came from stdin, send it again the same way." + ) + + def test_secret_on_argv_nulls_next_step(self, monkeypatch): + result = _pre(["--api-key", "sk_sentinel_123", "speak", "hi"], monkeypatch) + assert result.next_step is None + # The text was an argument: never ask for the key to be put back. + out_file = result.hint.split("--out-file ")[1].split(" ")[0] + assert OUT_FILE.match(out_file) + assert result.hint == ( + f"Run the same command again with --out-file {out_file} added. If " + "the text came from stdin, send it again the same way." + ) + assert "sk_sentinel_123" not in result.model_dump_json() + + @pytest.mark.parametrize( + "argv", + [ + ["--api-key=sk_x", "speak", "hi"], + ["--base-url", "https://user:pw@example.com", "speak", "hi"], + ["--base-url", "https://example.com/?token=x", "speak", "hi"], + ], + ) + def test_every_secret_form_nulls_next_step(self, monkeypatch, argv): + result = _pre(argv, monkeypatch) + assert result.next_step is None + assert result.hint.startswith("Run the same command again") + + @pytest.mark.parametrize( + "text", + ["Q&A starts now", "a < b; echo $HOME | x", "It's Q & A time", "I'll sudo it"], + ) + def test_shell_metacharacters_stay_inside_one_quoted_argument( + self, monkeypatch, text + ): + result = _pre(["speak", text], monkeypatch, text=text) + assert_e10(result.next_step) + assert shlex.split(result.next_step)[2] == text + + def test_a_token_starting_with_equals_nulls_next_step(self, monkeypatch): + result = _pre(["speak", "=cmd"], monkeypatch, text="=cmd") + assert result.next_step is None + assert result.hint.startswith("Run the same command again") + + @pytest.mark.parametrize( + "kw", [{"play": True}, {"list_voices": True}, {"out_file": "a.wav"}] + ) + def test_exempt(self, monkeypatch, kw): + assert _pre(["speak", "hi"], monkeypatch, **kw) is None + + def test_regular_file_tty_or_dev_null_is_allowed(self, monkeypatch): + monkeypatch.setattr(sinks, "stdout_takes_audio", lambda: True) + assert _pre(["speak", "hi"], monkeypatch) is None + + def test_people_and_ci_may_pipe_audio(self, monkeypatch): + monkeypatch.setattr(speak, "is_ai_agent", lambda: False) + assert _pre(["speak", "hi"], monkeypatch) is None + + +class TestStdoutTakesAudio: + def test_classification(self, monkeypatch, tmp_path): + monkeypatch.setattr(sinks, "stdout_takes_audio", _REAL_TAKES_AUDIO) + f = tmp_path / "out.wav" + f.write_bytes(b"") + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(f)) + assert _REAL_TAKES_AUDIO() is True + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.stat(os.devnull)) + assert _REAL_TAKES_AUDIO() is True + r, w = os.pipe() + try: + monkeypatch.setattr(sinks, "stdout_stat", lambda: os.fstat(w)) + monkeypatch.setattr(sinks.os, "isatty", lambda fd: False) + assert _REAL_TAKES_AUDIO() is False + monkeypatch.setattr(sinks.os, "isatty", lambda fd: True) + assert _REAL_TAKES_AUDIO() is True + finally: + os.close(r) + os.close(w) + monkeypatch.setattr(sinks, "stdout_stat", lambda: None) + assert _REAL_TAKES_AUDIO() is False + + +# ── Every speak coded error's next_step ───────────────────────────────── + + +class TestNextStepRows: + @pytest.mark.parametrize( + ("argv", "kw", "expected"), + [ + ( + ["speak", "hi", "-o", "a", "--out-file", "b"], + {"output": "a", "out_file": "b"}, + "dg speak --help", + ), + (["speak"], {"text": None}, "dg speak --help"), + (["speak", "hi"], {}, "rerun"), + (["speak", "-"], {"text": "-"}, None), + (["speak", "hi", "-o", "json"], {"output": "json"}, "rerun"), + (["speak", "hi", "-o", "text"], {"output": "text"}, "rerun"), + ( + ["speak", "hi", "--out-file", "nope/a.wav"], + {"out_file": "nope/a.wav"}, + None, + ), + ], + ) + def test_next_step_is_runnable_or_null(self, monkeypatch, argv, kw, expected): + result = _pre(argv, monkeypatch, **kw) + assert isinstance(result, SpeakCodedError) + assert_e10(result.next_step) + if expected is None: + assert result.next_step is None + elif expected == "rerun": + assert "--out-file" in shlex.split(result.next_step) + else: + assert result.next_step == expected + # hint, code and schema_version are never null; details omitted + # when empty. + dump = result.model_dump() + assert dump["hint"] and dump["code"] and dump["schema_version"] == 1 + assert list(dump)[:6] == [ + "error", + "status", + "code", + "hint", + "next_step", + "schema_version", + ] diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_strings.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_strings.py new file mode 100644 index 00000000..433db9ee --- /dev/null +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_strings.py @@ -0,0 +1,62 @@ +"""Speak's messages match the error catalogue (``error_codes.json``).""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path + +import pytest +from deepctl_cmd_speak import messages as msg + +# (code, error ID, hint_human, hint_agent, hint_agent_if_next_step_null) +_WIRING = [ + ("output_unwritable", "E-OUTPUT-UNWRITABLE", "H-OUTPUT-UNWRITABLE", + "A-OUTPUT-UNWRITABLE", None), + ("invalid_argument", "E-STDOUT-BYTES", "H-STDOUT-BYTES", "H-STDOUT-BYTES", None), + ("invalid_argument", "E-SPEAK-NO-OUTPUT", "H-SPEAK-NO-OUTPUT", + "A-SPEAK-NO-OUTPUT", "A-SPEAK-NO-OUTPUT-NULL"), + ("invalid_argument", "E-SPEAK-NO-TEXT", "H-SPEAK-NO-TEXT", "H-SPEAK-NO-TEXT", None), + ("invalid_argument", "E-OUT-FILE-EXISTS", "H-OUT-FILE-EXISTS", + "H-OUT-FILE-EXISTS", None), + ("invalid_argument", "E-SPEAK-O-FORMAT", "H-SPEAK-O-FORMAT", + "H-SPEAK-O-FORMAT", None), + ("invalid_argument", "E-SPEAK-O-TEXT", "H-SPEAK-O-TEXT", "H-SPEAK-O-TEXT", None), + ("invalid_argument", "E-SPEAK-OUT-CONFLICT", "H-SPEAK-OUT-CONFLICT", + "H-SPEAK-OUT-CONFLICT", None), +] # fmt: skip + + +def _ids() -> dict[str, str]: + return { + name.replace("_", "-"): value + for name, value in vars(msg).items() + if re.fullmatch(r"[EHAD]_[A-Z0-9_]+", name) + } + + +def test_every_wired_id_has_a_string(): + ids = _ids() + assert "A-SPEAK-NO-OUTPUT-ARGV" not in ids # folded into the NULL variant + for _code, *message_ids in _WIRING: + for message_id in message_ids: + assert message_id is None or message_id in ids, message_id + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_strings_match_catalogue(): + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + for message_id, text in _ids().items(): + assert catalogue["strings"][message_id] == text, message_id + for code, error_id, human, agent, agent_null in _WIRING: + entry = catalogue["codes"][code] + message = next(m for m in entry["messages"] if m["id"] == error_id) + assert message["hint_human"] == human, error_id + assert message["hint_agent"] == agent, error_id + assert message.get("hint_agent_if_next_step_null") == agent_null, error_id + assert msg.CODE_INVALID_ARGUMENT in catalogue["codes"] + assert msg.CODE_OUTPUT_UNWRITABLE in catalogue["codes"] diff --git a/src/deepctl/_manifest.d/speak.json b/src/deepctl/_manifest.d/speak.json index e6469999..b91768e1 100644 --- a/src/deepctl/_manifest.d/speak.json +++ b/src/deepctl/_manifest.d/speak.json @@ -8,7 +8,7 @@ "requires_auth": true, "help": { "name": "speak", - "help": "Convert text to speech using Deepgram TTS\n\nExamples:\n dg speak \"Hello from Deepgram\" --play\n dg speak --list-voices\n dg speak \"Hello world\"\n dg speak \"Hello world\" --out-file hello.wav\n dg speak \"Hello world\" --out-file hello.wav --play\n dg speak --file message.txt --out-file output.wav\n dg speak \"Hello\" | ffplay -loglevel error -nodisp -autoexit -\n dg speak \"A little slower, please\" --speed 0.9 --out-file slow.wav\n dg speak \"So exciting!\" --expressivity 2 --out-file lively.wav # beta; default: 0\n dg speak \"Hello\" -m aura-2-asteria-en --out-file hello.mp3\n dg speak \"Hello\" -m aura-2-luna-en --out-file hello.wav --encoding linear16 --container wav\n echo \"Hello\" | dg speak --out-file hello.mp3 -m aura-2-asteria-en\n dg speak \"Hola, mundo\" -m aura-2-selena-es --out-file hola.mp3\n\n\nFor agents, use --agent-friendly.", + "help": "Convert text to speech using Deepgram TTS\n\nExamples:\n dg speak \"Hello from Deepgram\" --play\n dg speak --list-voices\n dg speak \"Hello world\"\n dg speak \"Hello world\" --out-file hello.wav\n dg speak \"Hello world\" --out-file hello.wav --play\n dg speak --file message.txt --out-file output.wav\n dg speak \"Hello\" | ffplay -loglevel error -nodisp -autoexit -\n dg speak \"A little slower, please\" --speed 0.9 --out-file slow.wav\n dg speak \"So exciting!\" --expressivity 2 --out-file lively.wav # beta; default: 0\n dg speak \"Hello\" -m aura-2-asteria-en --out-file hello.mp3\n dg speak \"Hello\" -m aura-2-luna-en --out-file hello.wav --encoding linear16 --container wav\n echo \"Hello\" | dg speak - --out-file hello.mp3 -m aura-2-asteria-en\n dg speak \"Hola, mundo\" -m aura-2-selena-es --out-file hola.mp3\n\n\nFor agents, use --agent-friendly.", "short_help": "Text-to-speech", "epilog": null, "hidden": false, From 6b8907d511f048abae0a8c1f6ce9a1438d4dd2e8 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:53:21 -0400 Subject: [PATCH 11/18] fix(listen): report a failed stdout write as an output error When stdout couldn't be written during a live stream (dg listen - > /dev/full), the receive loop took the ENOSPC for a dropped connection: the run printed a network hint, then failed again flushing stdout at exit and ended with "Exception ignored" and exit 120. A stdout write error other than EPIPE, in the receive loop, while flushing Flux's last turns, or while writing the final result, is now output_unwritable: fd 1 is pointed at the null device, the error and hint go to stderr, the finals received so far travel in details, and the run exits 1. EPIPE still ends quietly. A failed write of a coded error also silences stdout, so the exit flush can't fail. The stdin catch-all now takes only read errors; an error of any other kind is reported without the stdin hint. Live listen now takes die_by_signal from deepctl_core.signals, the one copy speak also uses. --- .../src/deepctl_cmd_listen/command.py | 66 +++++- .../src/deepctl_cmd_listen/live.py | 52 ++--- .../src/deepctl_cmd_listen/messages.py | 15 ++ .../tests/unit/test_listen_output.py | 189 ++++++++++++++++++ .../tests/unit/test_live.py | 40 ++++ 5 files changed, 333 insertions(+), 29 deletions(-) create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_listen_output.py diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 01a48ed7..a1828a3d 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -188,6 +188,10 @@ class ListenCommand(BaseCommand): _ws_host: str = "" #: Set after a SIGTERM drain; exit_code_for re-raises it. _reraise_signal: int | None = None + #: The finals kept by the last live session, for a late stdout failure. + _live_kept: dict[str, Any] | None = None + #: Writing the final result to stdout failed; the run exits 1. + _output_failed: bool = False examples = [ "dg listen audio.mp3", @@ -1232,6 +1236,10 @@ def _finish_live( self._flush_v2(v2_state, transcript_acc, caption_writer=caption_writer) except BrokenPipeError: live.silence_stdout() + except OSError as exc: + live.silence_stdout() + if outcome.status == "success": + outcome = live.output_failed(exc) if outcome.status == "cancelled": status.print("\n[yellow]Stopped.[/yellow]") @@ -1266,6 +1274,7 @@ def kept() -> dict[str, Any]: out["saved_to"] = saved_to return out + self._live_kept = kept() if outcome.status == "drain_timeout": limit = round(outcome.drain_limit_s, 3) template = ( @@ -1311,7 +1320,9 @@ def kept() -> dict[str, Any]: }, ) - if outcome.status == "error": + if outcome.status == "error" and outcome.kind == "output": + return self._output_unwritable(outcome.os_error or "", kept()) + if outcome.status == "error" and outcome.kind == "read": # Reading stdin failed (EISDIR, EBADF, EIO on a hang-up). The # fix is the same as for a missing source: pass a file or URL. return self._coded_failure( @@ -1323,6 +1334,16 @@ def kept() -> dict[str, Any]: next_step=f"{_prog()} {self.name} --help", ) + if outcome.status == "error": + # An error of no known kind: report it without a stdin hint. + return self._coded_failure( + outcome.error or msg.E_LISTEN_CONNECTION_LOST, + code=msg.CODE_NETWORK_UNAVAILABLE, + hint_human=msg.H_NETWORK, + hint_agent=msg.A_NETWORK.format(prog=_prog()), + details={"host": self._ws_host, "reason": "connection lost", **kept()}, + ) + result = ListenResult( status="success", source=source, @@ -1361,14 +1382,30 @@ def _coded_failure( error=text, code=code, hint=hint, next_step=next_step, details=details ) + def _output_unwritable( + self, os_error: str, kept: dict[str, Any] + ) -> ListenCodedError: + """stdout couldn't be written: fd 1 is already silenced, + so the envelope goes to stderr only, as its two lines.""" + return self._coded_failure( + msg.E_OUTPUT_UNWRITABLE.format(output="stdout", reason=os_error), + code=msg.CODE_OUTPUT_UNWRITABLE, + hint_human=msg.H_OUTPUT_UNWRITABLE, + hint_agent=msg.A_OUTPUT_UNWRITABLE.format(prog=_prog(), reason=os_error), + details={"path": "-", "reason": os_error, **kept}, + ) + def exit_code_for(self, result: Any) -> int: """A SIGTERM-drained stream ends by SIGTERM itself: the - result is already written, so flush and re-raise it here.""" + result is already written, so flush and re-raise it here. A final + result that couldn't be written to stdout exits 1.""" if self._reraise_signal is not None: signum, self._reraise_signal = self._reraise_signal, None from . import live live.die_by_signal(signum) + if self._output_failed: + return 1 return super().exit_code_for(result) # ── Shared WebSocket helpers ─────────────────────────────────────── @@ -1512,7 +1549,12 @@ def _print_final(self, line: str) -> None: print(line, flush=True) else: if self._verbose: - print_info(escape(line), stderr=True) + try: + print_info(escape(line), stderr=True) + except OSError: + # A failing stderr is not a failing stdout: keep + # streaming, and keep the result that is the payload. + pass def _print_interim(self, text: str) -> None: """An interim rewrite; dropped under a machine format.""" @@ -1686,10 +1728,26 @@ def _default(obj: Any) -> str: except BrokenPipeError: # The reader went away (`| true`): end quietly, exit as is. live.silence_stdout() + except OSError as exc: + # stdout can't take the result (ENOSPC, EIO): output_unwritable + # on stderr, exit 1. + live.silence_stdout() + reason = live.output_failed(exc).os_error or "" + kept = self._live_kept or { + "transcript": result.transcript, + "transcript_chars": len(result.transcript), + } + self._output_unwritable(reason, kept) + self._output_failed = True return if not isinstance(result, ListenResult) or result.status != "success": - super().output_result(result, config) + try: + super().output_result(result, config) + except OSError: + # Already reported on stderr; keep the exit flush from + # failing again on a full or broken stdout. + live.silence_stdout() return # Caption mode: print captions to stdout regardless of output format diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py index 85f743c0..32d28e92 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/live.py @@ -38,6 +38,7 @@ from typing import TYPE_CHECKING, Any import click +from deepctl_core.signals import die_by_signal as die_by_signal # re-export from . import messages as msg @@ -230,8 +231,11 @@ class LiveOutcome: #: Error text for "error" (already scrubbed and safe to print). error: str | None = None #: For "error": "server" (error close or error frame), "network" (the - #: connection dropped) or "read" (stdin failed). + #: connection dropped), "read" (stdin failed) or "output" (stdout + #: couldn't be written; fd 1 now points at the null device). kind: str | None = None + #: For an "output" error: the OS error text (strerror), cleaned. + os_error: str | None = None #: The server's close code, for a "server" error that closed the socket. close_code: int | None = None #: The request id the server reported, when it sent one. @@ -336,6 +340,18 @@ def silence_stdout() -> None: pass +def output_failed(exc: OSError, secrets: tuple[str, ...] = ()) -> LiveOutcome: + """The outcome for a stdout write that failed with something other + than EPIPE.""" + reason = _clean(exc.strerror or exc, 200, secrets) or exc.__class__.__name__ + return LiveOutcome( + status="error", + kind="output", + error=msg.E_OUTPUT_UNWRITABLE.format(output="stdout", reason=reason), + os_error=reason, + ) + + def _is_connection_closed(exc: BaseException) -> bool: try: from websockets.exceptions import ConnectionClosed @@ -455,29 +471,6 @@ def restore(self) -> None: self._installed = False -def die_by_signal(signum: int) -> None: - """End the process by ``signum`` with its default action, so the wait - status is "killed by signal N" (shells report 128+N). - - PID 1 in a container ignores a default-action signal, so if the process - is still here afterwards it exits with the shell's 128+N. - """ - import sys - - for stream in (sys.stdout, sys.stderr): - try: - stream.flush() - except Exception: - pass - try: - signal.signal(signum, signal.SIG_DFL) - os.kill(os.getpid(), signum) - time.sleep(0.05) - except Exception: - pass - os._exit(128 + signum) - - async def _cancel_all(*tasks: asyncio.Task[Any] | None) -> None: live = [t for t in tasks if t is not None] for task in live: @@ -554,7 +547,16 @@ async def receive() -> LiveOutcome | None: try: async for raw in ws: track(raw) - on_message(raw) + try: + on_message(raw) + except BrokenPipeError: + raise + except OSError as exc: + # stdout can't be written (ENOSPC, EIO, ...): an output + # failure, not a network one. Silence fd 1 so the exit + # flush can't fail again. + silence_stdout() + return output_failed(exc, secrets) except BrokenPipeError: # Whoever read stdout is gone (``| head -n 1``): end quietly. silence_stdout() diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py index f8d51f3e..9d812f79 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/messages.py @@ -12,6 +12,7 @@ CODE_STREAM_DRAIN_TIMEOUT = "stream_drain_timeout" CODE_SERVER_ERROR = "server_error" CODE_NETWORK_UNAVAILABLE = "network_unavailable" +CODE_OUTPUT_UNWRITABLE = "output_unwritable" # invalid_argument: a hard agent signal, no SOURCE, no --mic. E_LISTEN_NO_SOURCE = ( @@ -83,3 +84,17 @@ # network_unavailable: the connection dropped without a closing handshake. E_LISTEN_CONNECTION_LOST = "The connection to the server was lost." + +# output_unwritable: writing the transcript or the final result to stdout +# failed with an error other than EPIPE (ENOSPC, EIO, ...). +# {reason} is details.reason. The texts are shared with speak. +E_OUTPUT_UNWRITABLE = "Couldn't write the output to {output}: {reason}." +H_OUTPUT_UNWRITABLE = ( + "Choose another destination, or fix this one (free space, permissions), " + "then try again." +) +A_OUTPUT_UNWRITABLE = ( + "If you chose this destination, choose another one. Otherwise tell the " + "user {prog} couldn't write its output ({reason}), and don't retry until " + "it's fixed." +) diff --git a/packages/deepctl-cmd-listen/tests/unit/test_listen_output.py b/packages/deepctl-cmd-listen/tests/unit/test_listen_output.py new file mode 100644 index 00000000..727a8af6 --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_listen_output.py @@ -0,0 +1,189 @@ +"""Live listen: a failed stdout write is output_unwritable, not a network +error, and the stdin catch-all only takes read errors.""" + +from __future__ import annotations + +import errno +import os +from unittest.mock import Mock + +import pytest +from deepctl_cmd_listen import command as listen_cmd +from deepctl_cmd_listen import live +from deepctl_cmd_listen.command import ListenCommand +from deepctl_cmd_listen.models import ListenCodedError, ListenResult +from deepctl_core import Config + + +@pytest.fixture +def command(monkeypatch): + monkeypatch.setattr(listen_cmd, "is_ai_agent", lambda: True) + return ListenCommand() + + +def _finish(command, outcome, **kw): + return command._finish_live( + outcome, + source="stdin", + model="nova-3", + language="en", + diarize=False, + transcript_acc=kw.get("acc", ["hello", "world"]), + caption_writer=None, + caption_format=None, + save_to=kw.get("save_to"), + v2_state=kw.get("v2_state"), + ) + + +def test_output_outcome_is_output_unwritable_with_the_kept_finals(command, capsys): + outcome = live.LiveOutcome( + status="error", + kind="output", + error="Couldn't write the output to stdout: No space left on device.", + os_error="No space left on device", + ) + result = _finish(command, outcome) + assert isinstance(result, ListenCodedError) + assert result.code == "output_unwritable" + assert ( + result.error == "Couldn't write the output to stdout: No space left on device." + ) + assert result.hint == ( + "If you chose this destination, choose another one. Otherwise tell the " + "user dg couldn't write its output (No space left on device), and don't " + "retry until it's fixed." + ) + assert result.next_step is None + assert result.details == { + "path": "-", + "reason": "No space left on device", + "transcript": "hello world", + "transcript_chars": 11, + } + assert command.exit_code_for(result) == 1 + err = capsys.readouterr().err + assert "network" not in err.lower() + + +def test_person_gets_the_human_hint(command, monkeypatch): + monkeypatch.setattr(listen_cmd, "is_ai_agent", lambda: False) + monkeypatch.setattr(listen_cmd, "_agentic", False) + outcome = live.LiveOutcome( + status="error", kind="output", os_error="Input/output error" + ) + result = _finish(command, outcome) + assert result.hint == ( + "Choose another destination, or fix this one (free space, permissions), " + "then try again." + ) + + +def test_stdin_catch_all_takes_read_errors_only(command): + read = _finish( + command, + live.LiveOutcome( + status="error", kind="read", error="Couldn't read audio from stdin: x." + ), + ) + assert read.code == "invalid_argument" + assert read.next_step == "dg listen --help" + other = _finish( + command, live.LiveOutcome(status="error", kind="mystery", error="boom") + ) + assert other.code == "network_unavailable" + assert other.next_step is None + + +def test_flux_flush_failure_is_output_unwritable(command, monkeypatch): + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + + def flush(*a, **kw): + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(command, "_flush_v2", flush) + result = _finish(command, live.LiveOutcome(), v2_state={"order": [], "turns": {}}) + assert result.code == "output_unwritable" + assert silenced == [True] + + +def test_final_result_write_failure_is_reported_and_exits_1( + command, monkeypatch, capsys +): + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + monkeypatch.setattr(listen_cmd, "get_output_format", lambda: "json") + stdout = Mock() + stdout.write.side_effect = OSError(errno.ENOSPC, "No space left on device") + monkeypatch.setattr(listen_cmd.sys, "stdout", stdout) + result = ListenResult(status="success", mode="live", transcript="hi there") + + command.output_result(result, Mock(spec=Config)) + + assert silenced == [True] + assert command.exit_code_for(result) == 1 + err = capsys.readouterr().err + assert "Couldn't write the output to stdout: No space left on device." in err + + +def test_final_result_epipe_stays_quiet(command, monkeypatch, capsys): + monkeypatch.setattr(live, "silence_stdout", lambda: None) + monkeypatch.setattr(listen_cmd, "get_output_format", lambda: "json") + stdout = Mock() + stdout.write.side_effect = BrokenPipeError(errno.EPIPE, "Broken pipe") + monkeypatch.setattr(listen_cmd.sys, "stdout", stdout) + result = ListenResult(status="success", mode="live", transcript="hi") + + command.output_result(result, Mock(spec=Config)) + + assert command.exit_code_for(result) == 0 + assert capsys.readouterr().err == "" + + +def test_coded_error_write_failure_silences_stdout(command, monkeypatch): + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + monkeypatch.setattr( + listen_cmd.BaseCommand, + "output_result", + Mock(side_effect=OSError(errno.ENOSPC, "No space left on device")), + ) + result = ListenCodedError(error="x", code="server_error", hint="h") + command.output_result(result, Mock(spec=Config)) + assert silenced == [True] + + +@pytest.mark.parametrize("exc", [OSError(errno.ENOSPC, "full"), BrokenPipeError()]) +def test_verbose_stderr_failure_is_not_a_stdout_failure(command, monkeypatch, exc): + """An INFO line that can't reach stderr leaves stdout and the + session alone.""" + command._echo = False + command._verbose = True + monkeypatch.setattr(listen_cmd, "print_info", Mock(side_effect=exc)) + command._print_final("hello") # does not raise + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_strings_match_catalogue(): + """Listen's messages, the output_unwritable texts shared with speak + included, match the error catalogue.""" + import json + import re + from pathlib import Path + + from deepctl_cmd_listen import messages as msg + + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + for name, text in vars(msg).items(): + if re.fullmatch(r"[EHAD]_[A-Z0-9_]+", name): + assert catalogue["strings"][name.replace("_", "-")] == text, name + message = catalogue["codes"]["output_unwritable"]["messages"][0] + assert (message["id"], message["hint_human"], message["hint_agent"]) == ( + "E-OUTPUT-UNWRITABLE", + "H-OUTPUT-UNWRITABLE", + "A-OUTPUT-UNWRITABLE", + ) diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live.py b/packages/deepctl-cmd-listen/tests/unit/test_live.py index f4a1e09e..e11603b4 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_live.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_live.py @@ -812,6 +812,46 @@ async def main(): assert silenced == [True] +@pytest.mark.parametrize( + ("err", "text"), + [ + (28, "No space left on device"), + (5, "Input/output error"), + (27, "File too large"), + ], +) +def test_other_stdout_write_errors_are_output_failures(monkeypatch, err, text): + """ENOSPC/EIO/EFBIG on stdout is not a network failure.""" + silenced = [] + monkeypatch.setattr(live, "silence_stdout", lambda: silenced.append(True)) + install_fake_websockets(monkeypatch, FakeWS([_result("a"), _result("b")])) + seen = [] + + def on_message(raw): + seen.append(raw) + raise OSError(err, text) + + async def main(): + return await live.run_session( + "wss://x", + {}, + on_message=on_message, + queue=asyncio.Queue(), + closing=threading.Event(), + start_source=lambda: None, + keepalive=False, + byte_rate=0, + ) + + outcome = run(main()) + assert outcome.status == "error" + assert outcome.kind == "output" + assert outcome.os_error == text + assert outcome.error == f"Couldn't write the output to stdout: {text}." + assert silenced == [True] + assert len(seen) == 1 # stopped at the first failed write + + # ── SIGTERM and non-blocking stdin ───────────────────────────────────────── From ad6f9f62202d5abe76973f1e5c30b7e8827d01f6 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 01:32:37 -0400 Subject: [PATCH 12/18] fix(cli): treat Ctrl-C while the CLI is loading as cancelled A Ctrl-C in the first few hundred milliseconds, while main() imports the CLI, reached the top level as KeyboardInterrupt and printed a traceback with exit 1. It is now a cancelled run, like a Ctrl-C once run() has started: the same "Operation cancelled by user" line on stderr, exit 2, and no traceback. main() also replaces Python's default SIGINT handler with one that holds a Ctrl-C back for a moment (at most 200 ms) when it lands in importlib's own code. A KeyboardInterrupt raised there can be swallowed and leave the import lock held, so the run then hangs on its next import. --- conftest.py | 11 +++++++++++ src/deepctl/main.py | 15 ++++++++++++++- tests/unit/test_entry_point.py | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 22734911..86d564bc 100644 --- a/conftest.py +++ b/conftest.py @@ -77,3 +77,14 @@ def _disable_telemetry(monkeypatch): """ monkeypatch.setenv("DEEPCTL_TELEMETRY_DISABLED", "1") monkeypatch.setenv("DEEPCTL_TELEMETRY_DSN", "http://k@127.0.0.1:9/1") + + +@pytest.fixture(autouse=True) +def _restore_sigint(): + """`main()` installs its own SIGINT handler; put pytest's back.""" + import signal + + before = signal.getsignal(signal.SIGINT) + yield + if before is not None: + signal.signal(signal.SIGINT, before) diff --git a/src/deepctl/main.py b/src/deepctl/main.py index f7695013..5e97e670 100644 --- a/src/deepctl/main.py +++ b/src/deepctl/main.py @@ -150,7 +150,20 @@ def main(prog: "str | None" = None) -> None: sys.exit(serve()) sys.excepthook = _rich_excepthook - from deepctl._cli import run + try: + from deepctl_core.signals import install_interrupt_handler + + install_interrupt_handler() + from deepctl._cli import run + except KeyboardInterrupt: + # Ctrl-C while the CLI is still loading: cancelled, exit 2, the same + # line run() prints (rich may not be loaded yet), and no traceback. + try: + sys.stderr.write("\nOperation cancelled by user\n") + sys.stderr.flush() + except (AttributeError, OSError, ValueError): + pass + sys.exit(2) run(args, started_at, prog) diff --git a/tests/unit/test_entry_point.py b/tests/unit/test_entry_point.py index 7490c51b..52d8cca2 100644 --- a/tests/unit/test_entry_point.py +++ b/tests/unit/test_entry_point.py @@ -253,3 +253,30 @@ def test_import_httpx_skips_its_cli(self, tmp_path: Path) -> None: proc = _run([sys.executable, "-c", code], tmp_path) assert proc.returncode == 0, proc.stderr assert proc.stderr.split()[-3:] == ["False", "True", "True"], proc.stderr + + +class TestInterruptWhileLoading: + def test_ctrl_c_while_the_cli_loads_is_cancelled_quietly( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """A Ctrl-C before run() is entered is cancelled: exit 2, run()'s + cancel line, no traceback.""" + from deepctl import main as entry + + class Interrupts: + def find_spec(self, name: str, *args: object) -> None: + if name == "deepctl._cli": + raise KeyboardInterrupt + return None + + monkeypatch.delitem(sys.modules, "deepctl._cli", raising=False) + monkeypatch.setattr(sys, "meta_path", [Interrupts(), *sys.meta_path]) + monkeypatch.setattr(sys, "argv", ["dg", "speak", "hi"]) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + with pytest.raises(SystemExit) as exc: + entry.main() + assert exc.value.code == 2 + captured = capsys.readouterr() + assert "Traceback" not in captured.err + assert captured.err == "\nOperation cancelled by user\n" + assert captured.out == "" From 9f34dbce18aa639bd39d95e8d03957dd6da06aec Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:23:32 -0400 Subject: [PATCH 13/18] perf(listen): stream prerecorded uploads from disk dg listen read the whole file into memory before sending it, so a 500 MB file peaked at 1.6 GB RSS. The file now goes to the SDK as a FileBody (deepctl_core/upload.py) with an explicit Content-Length, read in 1 MiB chunks as httpx sends them. httpx sends no chunked encoding. - FileBody opens the file once and keeps the descriptor. Each pass reads from offset 0 of that descriptor, so the SDK's retries (connect errors, a connection dropped before the response, 408/409/429/5xx) resend the whole file, and a file swapped in at the same path between retries is never sent. - The file validated is the file uploaded: listen takes the file's (st_dev, st_ino) before validating it, and FileBody compares the opened descriptor against it. A path swapped in between stops with E-UPLOAD-FILE-CHANGED before anything is sent. - Before each chunk, and after the last, the descriptor's size and mtime are compared with the values at open. A change stops the upload with invalid_argument E-UPLOAD-FILE-CHANGED; a failed read with E-UPLOAD-FILE-UNREADABLE. Both print the two stderr lines and the coded result a transport error does, with next_step null and details.path. - A pipe or device (dg listen <(...) --no-validate) is read once at open, as before, so a retry can resend it. - The descriptor closes when the request finishes. --- .../src/deepctl_cmd_listen/command.py | 23 +- .../tests/unit/test_prerecorded_upload.py | 95 +++++ .../deepctl-core/src/deepctl_core/client.py | 27 +- .../deepctl-core/src/deepctl_core/upload.py | 210 ++++++++++ .../deepctl-core/tests/unit/test_client.py | 19 +- .../deepctl-core/tests/unit/test_upload.py | 359 ++++++++++++++++++ 6 files changed, 715 insertions(+), 18 deletions(-) create mode 100644 packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py create mode 100644 packages/deepctl-core/src/deepctl_core/upload.py create mode 100644 packages/deepctl-core/tests/unit/test_upload.py diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index a1828a3d..73449218 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -27,6 +27,7 @@ AuthManager, BaseCommand, BaseResult, + CodedErrorResult, Config, DeepgramClient, ) @@ -412,7 +413,7 @@ def handle( auth_manager: AuthManager, client: DeepgramClient, **kwargs: Any, - ) -> BaseResult | ListenCodedError: + ) -> BaseResult | CodedErrorResult: source: str | None = kwargs.get("source") use_mic: bool = kwargs.get("mic", False) guided_flow = False @@ -697,7 +698,7 @@ def _prerecorded( no_validate: bool, caption_format: str | None, config: Config, - ) -> BaseResult: + ) -> BaseResult | CodedErrorResult: from deepctl_shared_utils import ( probe_file, require_ffprobe, @@ -707,6 +708,18 @@ def _prerecorded( source_kind = "url" if is_url else "file" + # The file validated (and probed) must be the file uploaded: its + # identity now, compared again once the upload opens it. + file_id: tuple[int, int] | None = None + if not is_url: + import os + + try: + st = os.stat(source) + file_id = (st.st_dev, st.st_ino) + except OSError: + pass # reported below (or by the upload, with --no-validate) + # ── Validation ───────────────────────────────────────────────── if not no_validate: if is_url: @@ -771,11 +784,15 @@ def _prerecorded( status.print(f"[dim]Transcribing[/dim] {source}") status.print(f"[dim]Model:[/dim] {model} [dim]Language:[/dim] {language}") + from deepctl_core.upload import UploadFileError, coded_result + try: if is_url: result_dict = client.transcribe_url(source, options) else: - result_dict = client.transcribe_file(source, options) + result_dict = client.transcribe_file(source, options, file_id=file_id) + except UploadFileError as e: + return coded_result(e) except Exception as e: # Raise (not return an error result) so an API rejection — e.g. a # --redact value the v1 endpoint refuses — exits non-zero and is diff --git a/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py new file mode 100644 index 00000000..b32751bf --- /dev/null +++ b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py @@ -0,0 +1,95 @@ +"""``dg listen`` prerecorded uploads (W2.4): streamed file bodies.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest.mock import Mock + +import pytest +from deepctl_cmd_listen.command import ListenCommand +from deepctl_core import Config, output, transport, upload +from deepctl_core.upload import UploadFileError + + +@pytest.fixture(autouse=True) +def _plain_output(monkeypatch): + monkeypatch.setitem(output._output_config, "agentic", False) + monkeypatch.setattr(transport, "prog", lambda: "dg") + + +def _prerecorded(client, source="audio.wav", **overrides): + kwargs: dict[str, Any] = { + "is_url": False, + "model": "nova-3", + "language": "en-US", + "api_version": 1, + "diarize": False, + "smart_format": True, + "punctuate": True, + "summarize": False, + "topics": False, + "sentiment": False, + "redact": (), + "numerals": False, + "save_to": None, + "probe": False, + "no_validate": True, + "caption_format": None, + "config": Mock(spec=Config), + } + kwargs.update(overrides) + return ListenCommand()._prerecorded(client, source, **kwargs) + + +def test_file_changed_returns_coded_error(capsys): + client = Mock() + client.transcribe_file.side_effect = UploadFileError( + upload.FILE_CHANGED, "audio.wav" + ) + result = _prerecorded(client) + payload = result.model_dump() + assert payload["status"] == "error" + assert payload["code"] == "invalid_argument" + assert payload["error"] == "audio.wav changed while dg was uploading it." + assert payload["next_step"] is None + assert payload["details"] == {"path": "audio.wav"} + assert ListenCommand().exit_code_for(result) == 1 + assert "changed while dg was uploading it" in capsys.readouterr().err + + +def test_validated_file_identity_reaches_the_upload(tmp_path): + audio = tmp_path / "a.wav" + audio.write_bytes(b"RIFF" + b"\0" * 64) + st = audio.stat() + client = Mock() + client.transcribe_file.return_value = {"results": {"channels": []}} + _prerecorded(client, str(audio), no_validate=False) + assert client.transcribe_file.call_args.kwargs["file_id"] == (st.st_dev, st.st_ino) + + +def test_path_swapped_during_validation_uploads_nothing(tmp_path, monkeypatch): + """The file validated is the file uploaded.""" + import deepctl_shared_utils + from deepctl_core.client import DeepgramClient + + audio = tmp_path / "a.wav" + audio.write_bytes(b"RIFF" + b"\0" * 64) + other = tmp_path / "other.wav" + other.write_bytes(b"RIFF" + b"\1" * 64) + + def validate_then_swap(path): + os.replace(other, audio) # someone swaps the path mid-validation + return True + + monkeypatch.setattr(deepctl_shared_utils, "validate_audio_file", validate_then_swap) + client = Mock(spec=DeepgramClient) + client.transcribe_file.side_effect = lambda path, options, **kw: ( + DeepgramClient.transcribe_file(client, path, options, **kw) + ) + client._secrets = () + result = _prerecorded(client, str(audio), no_validate=False) + payload = result.model_dump() + assert payload["code"] == "invalid_argument" + assert payload["error"] == f"{audio} changed while dg was uploading it." + client.client.listen.v1.media.transcribe_file.assert_not_called() diff --git a/packages/deepctl-core/src/deepctl_core/client.py b/packages/deepctl-core/src/deepctl_core/client.py index 19399c33..bbfb0d4a 100644 --- a/packages/deepctl-core/src/deepctl_core/client.py +++ b/packages/deepctl-core/src/deepctl_core/client.py @@ -202,8 +202,14 @@ def transcribe_file( self, file_path: str | Path, options: dict[str, Any] | None = None, + *, + file_id: tuple[int, int] | None = None, ) -> dict[str, Any]: - """Transcribe an audio file.""" + """Transcribe an audio file. + + ``file_id`` is ``(st_dev, st_ino)`` of the file the caller validated; + the upload stops if the path now names another file. + """ file_path = Path(file_path) if not file_path.exists(): raise FileNotFoundError(f"Audio file not found: {file_path}") @@ -216,15 +222,22 @@ def transcribe_file( if options: default_options.update(options) - try: - with open(file_path, "rb") as audio_file: - audio_data = audio_file.read() + from .upload import FileBody, UploadFileError - response = self.client.listen.v1.media.transcribe_file( - request=audio_data, **default_options - ) + try: + # Streamed from disk; the SDK resends the same body on a retry. + with FileBody(file_path, file_id=file_id) as body: + response = self.client.listen.v1.media.transcribe_file( + request=body, + request_options={ + "additional_headers": {"Content-Length": str(len(body))} + }, + **default_options, + ) return cast("dict[str, Any]", response.model_dump()) + except UploadFileError: + raise except Exception as e: transport.raise_for(e, self._secrets) raise self._failed("Transcription failed", e) diff --git a/packages/deepctl-core/src/deepctl_core/upload.py b/packages/deepctl-core/src/deepctl_core/upload.py new file mode 100644 index 00000000..7ad3e2e0 --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/upload.py @@ -0,0 +1,210 @@ +"""Request bodies read from disk while they are sent. + +:class:`FileBody` streams a local file to the SDK in 1 MiB chunks instead of +reading it into memory. It opens the file once and keeps the descriptor: +every pass over the body reads from offset 0 of that descriptor, so the +SDK's retries (it sends the same content object again) resend the same +file, and a file swapped in at the path between retries is never sent. + +Before each chunk the descriptor's size and mtime are compared with the +values at open. A change raises :class:`UploadFileError` +(``E-UPLOAD-FILE-CHANGED``); a failed read raises it with +``E-UPLOAD-FILE-UNREADABLE``. Both are ``invalid_argument``: the user fixes +the input. :func:`coded_result` reports one the way a transport error is +reported. +""" + +from __future__ import annotations + +import os +import stat +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from .models import CodedErrorResult + +CHUNK_SIZE = 1 << 20 + +# ── Catalogue text (error_codes.json "strings") ────────────────────── + +STRINGS: dict[str, str] = { + "E-UPLOAD-FILE-CHANGED": "{path} changed while {prog} was uploading it.", + "H-UPLOAD-FILE-CHANGED": ( + "Wait until the file is completely written, then try again." + ), + "A-UPLOAD-FILE-CHANGED": ( + "The file was still being written. Wait for it to finish, then retry once." + ), + "E-UPLOAD-FILE-UNREADABLE": "Couldn't read {path}: {os_error}.", + "H-UPLOAD-FILE-UNREADABLE": ( + "Check that the file still exists and is readable, then try again." + ), +} + +# (error message ID, human hint, agent hint) +Messages = tuple[str, str, str] +FILE_CHANGED: Messages = ( + "E-UPLOAD-FILE-CHANGED", + "H-UPLOAD-FILE-CHANGED", + "A-UPLOAD-FILE-CHANGED", +) +FILE_UNREADABLE: Messages = ( + "E-UPLOAD-FILE-UNREADABLE", + "H-UPLOAD-FILE-UNREADABLE", + "H-UPLOAD-FILE-UNREADABLE", +) + + +class UploadFileError(Exception): + """The file being uploaded changed, or couldn't be read.""" + + code = "invalid_argument" + + def __init__( + self, messages: Messages, path: str, os_error: OSError | None = None + ) -> None: + from .transport import _clean + + self.messages = messages + self.path = _clean(path) + self.values: dict[str, str] = {"path": self.path} + if os_error is not None: + reason = os_error.strerror or type(os_error).__name__ + self.values["os_error"] = _clean(reason) + super().__init__(STRINGS[messages[0]].format(prog="dg", **self.values)) + + +class FileBody: + """A local file as a re-iterable request body with a known length. + + Give it to the SDK as ``request=`` with a ``Content-Length`` header of + ``len(body)``; httpx then sends no ``Transfer-Encoding: chunked``. Use + it as a context manager, or call :meth:`close`, to close the descriptor. + It has no ``read`` method on purpose: httpx would read it once, as a + file, and a retry would send nothing. + """ + + def __init__( + self, path: str | Path, *, file_id: tuple[int, int] | None = None + ) -> None: + """Open ``path``. + + ``file_id`` is ``(st_dev, st_ino)`` from a stat taken when the path + was validated: if the path now names another file, the upload stops + with ``E-UPLOAD-FILE-CHANGED`` before anything is sent. + """ + self.path = os.fspath(path) + self._data: bytes | None = None + try: + self._fd: int | None = os.open( + self.path, os.O_RDONLY | getattr(os, "O_BINARY", 0) + ) + except OSError as e: + raise UploadFileError(FILE_UNREADABLE, self.path, e) from e + try: + st = os.fstat(self._fd) + if file_id is not None and (st.st_dev, st.st_ino) != file_id: + self.close() + raise UploadFileError(FILE_CHANGED, self.path) + if not stat.S_ISREG(st.st_mode): + # A pipe or device (``<(...)``) can be read only once: keep + # its bytes, as 0.4 did, so a retry can resend them. + self._data = self._read_all() + self._size = len(self._data) + self.close() + return + except OSError as e: + self.close() + raise UploadFileError(FILE_UNREADABLE, self.path, e) from e + self._size = st.st_size + self._stamp = (st.st_size, st.st_mtime_ns) + + def __len__(self) -> int: + return self._size + + def __iter__(self) -> Iterator[bytes]: + if self._data is not None: + return iter((self._data,)) + return self._chunks() + + def __enter__(self) -> FileBody: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + fd, self._fd = self._fd, None + if fd is not None: + os.close(fd) + + def _chunks(self) -> Iterator[bytes]: + offset = 0 + while offset < self._size: + self._check() + data = self._pread(min(CHUNK_SIZE, self._size - offset), offset) + if not data: # shorter than at open + raise UploadFileError(FILE_CHANGED, self.path) + yield data + offset += len(data) + self._check() + + def _check(self) -> None: + if self._fd is None: + raise UploadFileError( + FILE_UNREADABLE, self.path, OSError(9, os.strerror(9)) + ) + try: + st = os.fstat(self._fd) + except OSError as e: + raise UploadFileError(FILE_UNREADABLE, self.path, e) from e + if (st.st_size, st.st_mtime_ns) != self._stamp: + raise UploadFileError(FILE_CHANGED, self.path) + + def _pread(self, size: int, offset: int) -> bytes: + assert self._fd is not None + try: + if hasattr(os, "pread"): + return os.pread(self._fd, size, offset) + # Windows has no pread; one pass reads at a time. + os.lseek(self._fd, offset, os.SEEK_SET) + return os.read(self._fd, size) + except OSError as e: + raise UploadFileError(FILE_UNREADABLE, self.path, e) from e + + def _read_all(self) -> bytes: + assert self._fd is not None + parts = [] + while chunk := os.read(self._fd, CHUNK_SIZE): + parts.append(chunk) + return b"".join(parts) + + +def coded_result(err: UploadFileError) -> CodedErrorResult: + """Report ``err`` on stderr (two lines) and return its coded result. + + The same shape as a transport error, with ``next_step`` null: no + command fixes a file that is still being written. + """ + from rich.markup import escape + + from . import output, transport + + program = transport.prog() + values = {**err.values, "prog": program or "dg"} + agentic = bool(output._output_config["agentic"]) + error = STRINGS[err.messages[0]].format(**values) + hint = STRINGS[err.messages[2 if agentic else 1]].format(**values) + output.print_error(escape(error)) + output.print_hint(escape(hint)) + result = transport.result_class()( + error=error, + code=err.code, + hint=hint, + next_step=None, + details={"path": err.path}, + ) + return cast("CodedErrorResult", result) diff --git a/packages/deepctl-core/tests/unit/test_client.py b/packages/deepctl-core/tests/unit/test_client.py index 17be91d2..c6bfeb68 100644 --- a/packages/deepctl-core/tests/unit/test_client.py +++ b/packages/deepctl-core/tests/unit/test_client.py @@ -1,7 +1,7 @@ """Tests for the DeepgramClient class.""" from pathlib import Path -from unittest.mock import MagicMock, Mock, mock_open, patch +from unittest.mock import MagicMock, Mock, patch import pytest from deepctl_core import transport @@ -176,12 +176,10 @@ def test_create_client_error_handling( client._create_client() @patch("deepctl_core.client.DGClient") - @patch("deepctl_core.client.Path.exists") - @patch("builtins.open", new_callable=mock_open, read_data=b"audio data") - def test_transcribe_file(self, mock_file, mock_exists, mock_dg_client, client): + def test_transcribe_file(self, mock_dg_client, client, tmp_path): """Test transcribing a file.""" - # Setup mocks - mock_exists.return_value = True + audio = tmp_path / "test.mp3" + audio.write_bytes(b"audio data") mock_instance = Mock() mock_media = Mock() mock_data = {"transcript": "Hello world"} @@ -191,13 +189,18 @@ def test_transcribe_file(self, mock_file, mock_exists, mock_dg_client, client): # Test transcription result = client.transcribe_file( - "/audio/test.mp3", options={"model": "nova-2", "language": "en"} + audio, options={"model": "nova-2", "language": "en"} ) assert result == {"transcript": "Hello world"} - # Verify transcribe_file was called + # Verify transcribe_file was called with a streamed body mock_media.transcribe_file.assert_called_once() + kwargs = mock_media.transcribe_file.call_args.kwargs + assert kwargs["request_options"] == { + "additional_headers": {"Content-Length": "10"} + } + assert kwargs["model"] == "nova-2" @patch("deepctl_core.client.DGClient") def test_transcribe_url(self, mock_dg_client, client): diff --git a/packages/deepctl-core/tests/unit/test_upload.py b/packages/deepctl-core/tests/unit/test_upload.py new file mode 100644 index 00000000..821ec303 --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_upload.py @@ -0,0 +1,359 @@ +"""Prerecorded uploads streamed from disk. + +``FileBody`` on its own, then through the real SDK facade on an +``httpx.MockTransport``: the request carries ``Content-Length`` and no +chunked encoding, the SDK's retries resend the whole file, and a file that +changes or can't be read mid-upload maps to the ``E-UPLOAD-*`` messages. +""" + +from __future__ import annotations + +import errno +import json +import os +import sys +import threading +from pathlib import Path + +import httpx +import pytest +from deepctl_core import hosts, output, transport, upload +from deepctl_core.auth import AuthManager +from deepctl_core.client import DeepgramClient +from deepctl_core.config import Config +from deepctl_core.upload import FileBody, UploadFileError + +KEY = "sk-URLSENTINELKEY" +MIB = 1 << 20 + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "deepctl_core.config._default_config_file", + lambda: tmp_path / "user" / "config.yaml", + ) + monkeypatch.setattr( + "deepctl_core.config._trust_root_location", + lambda: (tmp_path, tmp_path / "user" / "config.yaml"), + ) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setattr(transport, "_client", None) + monkeypatch.setattr(transport, "prog", lambda: "dg") + monkeypatch.setitem(output._output_config, "agentic", False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + from deepctl_core import auth as auth_module + + monkeypatch.setattr(auth_module.keyring, "get_password", lambda *a: None) + import deepgram.core.http_client as sdk_http + + monkeypatch.setattr(sdk_http.time, "sleep", lambda s: None) + monkeypatch.setenv("DEEPGRAM_API_KEY", KEY) + + +def _audio(tmp_path: Path, size: int, name: str = "a.wav") -> Path: + path = tmp_path / name + path.write_bytes(bytes(i % 251 for i in range(size))) + return path + + +def facade(handler) -> tuple[DeepgramClient, list[tuple[httpx.Request, bytes]]]: + seen: list[tuple[httpx.Request, bytes]] = [] + + def record(request: httpx.Request) -> httpx.Response: + seen.append((request, request.read())) + return handler(request) + + transport._set_client( + httpx.Client(transport=httpx.MockTransport(record), follow_redirects=True) + ) + config = Config(config_path=str(Path.cwd() / "none.yaml")) + return DeepgramClient(config, AuthManager(config)), seen + + +def ok(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"metadata": {}, "results": {}}) + + +# ── FileBody ───────────────────────────────────────────────────────── + + +def test_reads_in_1_mib_chunks_and_reiterates(tmp_path): + path = _audio(tmp_path, 2 * MIB + 5) + with FileBody(path) as body: + assert len(body) == 2 * MIB + 5 + first = list(body) + assert [len(c) for c in first] == [MIB, MIB, 5] + assert b"".join(body) == path.read_bytes() + + +def test_a_retry_sends_the_file_opened_not_the_one_at_the_path(tmp_path): + path = _audio(tmp_path, 3000) + original = path.read_bytes() + with FileBody(path) as body: + assert b"".join(body) == original + swap = tmp_path / "swap.wav" + swap.write_bytes(b"x" * 3000) + os.replace(swap, path) # a new file at the same path + assert b"".join(body) == original + + +def test_file_changed_mid_read(tmp_path): + path = _audio(tmp_path, 2 * MIB) + with FileBody(path) as body: + chunks = iter(body) + next(chunks) + os.truncate(path, MIB // 2) + with pytest.raises(UploadFileError) as err: + next(chunks) + assert err.value.messages == upload.FILE_CHANGED + assert err.value.values == {"path": str(path)} + + +def test_file_grown_after_the_last_chunk(tmp_path): + path = _audio(tmp_path, 10) + with FileBody(path) as body: + chunks = iter(body) + next(chunks) + with path.open("ab") as f: + f.write(b"more") + with pytest.raises(UploadFileError) as err: + next(chunks) + assert err.value.messages == upload.FILE_CHANGED + + +def test_read_error_is_unreadable(tmp_path, monkeypatch): + path = _audio(tmp_path, 10) + + def eio(*args): + raise OSError(errno.EIO, os.strerror(errno.EIO)) + + with FileBody(path) as body: + monkeypatch.setattr(os, "pread", eio, raising=False) + with pytest.raises(UploadFileError) as err: + list(body) + assert err.value.messages == upload.FILE_UNREADABLE + assert err.value.values["os_error"] == os.strerror(errno.EIO) + + +def test_missing_file_is_unreadable(tmp_path): + with pytest.raises(UploadFileError) as err: + FileBody(tmp_path / "gone.wav") + assert err.value.messages == upload.FILE_UNREADABLE + assert err.value.values["os_error"] == os.strerror(errno.ENOENT) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="needs a FIFO") +def test_a_pipe_is_read_once_and_kept(tmp_path): + fifo = tmp_path / "audio.fifo" + os.mkfifo(fifo) + writer = threading.Thread(target=lambda: fifo.write_bytes(b"piped audio")) + writer.start() + with FileBody(fifo) as body: + writer.join() + assert len(body) == 11 + assert b"".join(body) == b"piped audio" + assert b"".join(body) == b"piped audio" + + +def _file_id(path: Path) -> tuple[int, int]: + st = os.stat(path) + return (st.st_dev, st.st_ino) + + +def test_same_file_as_validated_is_uploaded(tmp_path): + path = _audio(tmp_path, 10) + with FileBody(path, file_id=_file_id(path)) as body: + assert b"".join(body) == path.read_bytes() + + +def test_path_swapped_after_validation_is_changed(tmp_path): + validated = _audio(tmp_path, 10, "validated.wav") + other = _audio(tmp_path, 10, "other.wav") + link = tmp_path / "x.wav" + try: + link.symlink_to(validated) + except OSError: + pytest.skip("symlinks not available") + file_id = _file_id(link) + link.unlink() + link.symlink_to(other) # swapped between validation and upload + with pytest.raises(UploadFileError) as err: + FileBody(link, file_id=file_id) + assert err.value.messages == upload.FILE_CHANGED + + +def test_close_releases_the_descriptor(tmp_path): + body = FileBody(_audio(tmp_path, 10)) + fd = body._fd + body.close() + body.close() + with pytest.raises(OSError): + os.fstat(fd) + + +def test_path_in_the_message_is_cleaned(tmp_path): + err = UploadFileError(upload.FILE_CHANGED, "/tmp/a\x1b[31m.wav") + assert "\x1b" not in err.values["path"] + + +# ── Through the SDK ────────────────────────────────────────────────── + + +def test_upload_has_content_length_and_no_chunked_encoding(tmp_path): + path = _audio(tmp_path, MIB + 7) + client, seen = facade(ok) + client.transcribe_file(path) + ((request, body),) = seen + assert request.headers["Content-Length"] == str(MIB + 7) + assert "Transfer-Encoding" not in request.headers + assert body == path.read_bytes() + + +def refused(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused", request=request) + + +def busy(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, json={"err_msg": "busy"}) + + +@pytest.mark.parametrize("first", [refused, busy], ids=["connect", "503"]) +def test_sdk_retry_resends_the_whole_file(tmp_path, first): + path = _audio(tmp_path, 2 * MIB + 3) + calls = [] + + def handler(request): + calls.append(1) + return first(request) if len(calls) == 1 else ok(request) + + client, seen = facade(handler) + client.transcribe_file(path) + assert len(seen) == 2 + assert [len(body) for _, body in seen] == [2 * MIB + 3] * 2 + assert seen[1][1] == path.read_bytes() + + +def test_swapped_path_sends_nothing(tmp_path): + validated = _audio(tmp_path, 10, "validated.wav") + file_id = _file_id(validated) + os.replace(_audio(tmp_path, 10, "other.wav"), validated) + client, seen = facade(ok) + with pytest.raises(UploadFileError) as err: + client.transcribe_file(validated, file_id=file_id) + assert err.value.messages == upload.FILE_CHANGED + assert seen == [] + + +def _changing_pread(path: Path, monkeypatch, action): + real = os.pread if hasattr(os, "pread") else None + calls = [] + + def pread(fd, size, offset): + calls.append(offset) + if len(calls) == 2: + action(path) + if real is not None: + return real(fd, size, offset) + os.lseek(fd, offset, os.SEEK_SET) + return os.read(fd, size) + + monkeypatch.setattr(os, "pread", pread, raising=False) + + +def test_file_changed_during_upload_maps_to_upload_changed(tmp_path, monkeypatch): + path = _audio(tmp_path, 3 * MIB) + client, seen = facade(ok) + _changing_pread(path, monkeypatch, lambda p: os.truncate(p, MIB)) + with pytest.raises(UploadFileError) as err: + client.transcribe_file(path) + assert err.value.messages == upload.FILE_CHANGED + assert str(err.value) == f"{path} changed while dg was uploading it." + assert seen == [] # the request never completed + + +def test_read_error_during_upload_maps_to_upload_unreadable(tmp_path, monkeypatch): + path = _audio(tmp_path, 3 * MIB) + client, _ = facade(ok) + real = getattr(os, "pread", None) + calls = [] + + def pread(fd, size, offset): + calls.append(offset) + if len(calls) == 2: + raise OSError(errno.EIO, os.strerror(errno.EIO)) + return real(fd, size, offset) if real else os.read(fd, size) + + monkeypatch.setattr(os, "pread", pread, raising=False) + with pytest.raises(UploadFileError) as err: + client.transcribe_file(path) + assert err.value.messages == upload.FILE_UNREADABLE + assert str(err.value) == f"Couldn't read {path}: {os.strerror(errno.EIO)}." + + +# ── The coded result ───────────────────────────────────────────────── + + +@pytest.mark.parametrize("agentic", [False, True]) +def test_coded_result(agentic, monkeypatch, capsys): + monkeypatch.setitem(output._output_config, "agentic", agentic) + err = UploadFileError(upload.FILE_CHANGED, "/data/call.wav") + payload = upload.coded_result(err).model_dump() + hint = upload.STRINGS[ + "A-UPLOAD-FILE-CHANGED" if agentic else "H-UPLOAD-FILE-CHANGED" + ] + assert payload == { + "error": "/data/call.wav changed while dg was uploading it.", + "status": "error", + "code": "invalid_argument", + "hint": hint, + "next_step": None, + "schema_version": 1, + "details": {"path": "/data/call.wav"}, + } + err_lines = capsys.readouterr().err + assert "/data/call.wav changed while dg was uploading it." in err_lines + assert hint in err_lines + + +def test_unreadable_uses_the_human_hint_for_agents(monkeypatch, capsys): + monkeypatch.setitem(output._output_config, "agentic", True) + err = UploadFileError( + upload.FILE_UNREADABLE, "/x.wav", OSError(errno.EACCES, "Permission denied") + ) + payload = upload.coded_result(err).model_dump() + assert payload["error"] == "Couldn't read /x.wav: Permission denied." + assert payload["hint"] == upload.STRINGS["H-UPLOAD-FILE-UNREADABLE"] + assert payload["next_step"] is None + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_strings_match_catalogue(): + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + for message_id, text in upload.STRINGS.items(): + assert catalogue["strings"][message_id] == text, message_id + entry = catalogue["codes"]["invalid_argument"] + assert "path" in entry["details"] + for error_id, human, agent in (upload.FILE_CHANGED, upload.FILE_UNREADABLE): + message = next(m for m in entry["messages"] if m["id"] == error_id) + assert (message["hint_human"], message["hint_agent"]) == (human, agent) + assert entry["next_step_overrides"][error_id] is None + + +def test_no_module_level_import_cost(): + """``deepctl_core.upload`` loads nothing heavier than ``os``.""" + code = ( + "import sys, deepctl_core.upload; " + "print(sorted(m for m in ('httpx', 'pydantic', 'rich', 'deepgram') " + "if m in sys.modules))" + ) + import subprocess + + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.strip() == "[]" From 773cef588cfa96094d7153d2a015342e3ae393f4 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:28:31 -0400 Subject: [PATCH 14/18] feat(listen): add --timeout for prerecorded transcription A prerecorded request used the shared client's 60 s timeout, so a large file, or a server that takes longer than a minute to answer, failed with a timeout even though the upload was fine. - dg listen --timeout SECONDS (at least 1, default 300) sets the request's timeout through the SDK's request_options. Every other command keeps 60 s. - E-TIMEOUT's details.timeout_s, and the "didn't respond within" text, report the effective value: transport.map_exception and raise_for take a keyword-only timeout_s. - --timeout with --mic, - or piped stdin is a usage error; live streams have no request timeout. --- .../src/deepctl_cmd_listen/command.py | 31 ++++++- .../tests/unit/test_prerecorded_upload.py | 87 ++++++++++++++++++- .../deepctl-core/src/deepctl_core/client.py | 39 ++++++--- .../src/deepctl_core/transport.py | 24 +++-- .../deepctl-core/tests/unit/test_upload.py | 51 ++++++++++- src/deepctl/_manifest.d/listen.json | 26 ++++++ src/deepctl/_manifest.d/transcribe.json | 26 ++++++ 7 files changed, 262 insertions(+), 22 deletions(-) diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 73449218..7ac3ecad 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -78,6 +78,11 @@ # vocabulary ("pci", "ssn", …) earns an opaque HTTP 400 from the v2 endpoint. _FLUX_REDACT = ("numbers", "aggressive_numbers") +# --timeout's default: a large file can take minutes to upload and process. +_PRERECORDED_TIMEOUT_S = 300.0 +# One instance, so `dg transcribe` (an alias) has equal arguments. +_TIMEOUT_TYPE = click.FloatRange(min=1) + def _is_url(s: str) -> bool: return s.startswith(("http://", "https://")) @@ -360,6 +365,16 @@ def get_arguments(self) -> list[dict[str, Any]]: "is_flag": True, "is_option": True, }, + { + "names": ["--timeout"], + "help": ( + "Seconds to wait at each step (connecting, sending, the " + "response) before giving up. Not a limit on the whole run. " + "Default: 300. Files and URLs only." + ), + "type": _TIMEOUT_TYPE, + "is_option": True, + }, # ── Caption output ──────────────────────────────────────── { "names": ["--webvtt"], @@ -496,6 +511,11 @@ def handle( save_to = kwargs.get("save_to") probe = kwargs.get("probe", False) no_validate = kwargs.get("no_validate", False) + timeout: float | None = kwargs.get("timeout") + if timeout is not None and mode in ("stream_mic", "stream_stdin"): + raise click.UsageError( + "--timeout applies only to a file or URL source, not to --mic or stdin." + ) # Flux STT (listen v2) is streaming-only — there is no v2 pre-recorded # REST endpoint, so a file/URL routes to /v1/listen and the server @@ -567,6 +587,7 @@ def handle( no_validate=no_validate, caption_format=caption_format, config=config, + timeout_s=timeout or _PRERECORDED_TIMEOUT_S, ) elif mode == "prerecorded_url": assert source is not None @@ -590,6 +611,7 @@ def handle( no_validate=no_validate, caption_format=caption_format, config=config, + timeout_s=timeout or _PRERECORDED_TIMEOUT_S, ) elif mode == "stream_mic": return self._stream_mic( @@ -698,6 +720,7 @@ def _prerecorded( no_validate: bool, caption_format: str | None, config: Config, + timeout_s: float = _PRERECORDED_TIMEOUT_S, ) -> BaseResult | CodedErrorResult: from deepctl_shared_utils import ( probe_file, @@ -788,9 +811,13 @@ def _prerecorded( try: if is_url: - result_dict = client.transcribe_url(source, options) + result_dict = client.transcribe_url( + source, options, timeout_s=timeout_s + ) else: - result_dict = client.transcribe_file(source, options, file_id=file_id) + result_dict = client.transcribe_file( + source, options, timeout_s=timeout_s, file_id=file_id + ) except UploadFileError as e: return coded_result(e) except Exception as e: diff --git a/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py index b32751bf..021d7f7a 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py @@ -1,14 +1,16 @@ -"""``dg listen`` prerecorded uploads (W2.4): streamed file bodies.""" +"""``dg listen`` prerecorded uploads (W2.4): streamed file bodies, --timeout.""" from __future__ import annotations import os from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest +from click.testing import CliRunner from deepctl_cmd_listen.command import ListenCommand -from deepctl_core import Config, output, transport, upload +from deepctl_cmd_listen.models import ListenResult +from deepctl_core import Config, PluginManager, output, transport, upload from deepctl_core.upload import UploadFileError @@ -93,3 +95,82 @@ def validate_then_swap(path): assert payload["code"] == "invalid_argument" assert payload["error"] == f"{audio} changed while dg was uploading it." client.client.listen.v1.media.transcribe_file.assert_not_called() + + +# ── --timeout ──────────────────────────────────────────────────────── + + +def _invoke(args: list[str], prerecorded: Mock | None = None, **streams: Mock): + config = Mock(spec=Config) + config.get.side_effect = lambda key, default=None: ( + True if key == "output.quiet" else default + ) + config.get_profile.return_value.base_url = "https://api.deepgram.com" + command = ListenCommand() + done = ListenResult(status="success", source="file", mode="prerecorded") + with ( + patch.object(command, "_prerecorded", prerecorded or Mock(return_value=done)), + patch.object(command, "_stream_mic", streams.get("mic", Mock())), + patch.object(command, "_stream_stdin", streams.get("stdin", Mock())), + patch("deepctl_core.base_command.AuthManager.guard"), + patch( + "deepctl_core.base_command.AuthManager.check_base_url", + return_value="https://api.deepgram.com", + ), + ): + click_command = PluginManager()._create_click_command(command) + return CliRunner().invoke(click_command, args, obj={"config": config}) + + +@pytest.mark.parametrize( + ("args", "expected"), + [ + (["a.wav"], 300.0), + (["a.wav", "--timeout", "120"], 120.0), + (["https://example.com/a.wav", "--timeout", "2.5"], 2.5), + ], +) +def test_timeout_reaches_the_request(args, expected): + done = ListenResult(status="success", source="file", mode="prerecorded") + prerecorded = Mock(return_value=done) + result = _invoke(args, prerecorded) + assert result.exit_code == 0, result.output + assert prerecorded.call_args.kwargs["timeout_s"] == expected + + +@pytest.mark.parametrize("value", ["0.5", "0", "-1", "soon"]) +def test_timeout_below_one_second_is_a_usage_error(value): + result = _invoke(["a.wav", "--timeout", value]) + assert result.exit_code == 2 + assert "--timeout" in result.output + + +@pytest.mark.parametrize("args", [["--mic"], ["-"]]) +def test_timeout_with_a_live_source_is_a_usage_error(args): + mic, stdin = Mock(), Mock() + result = _invoke([*args, "--timeout", "30"], mic=mic, stdin=stdin) + assert result.exit_code == 2 + assert "--timeout applies only to a file or URL source" in result.output + mic.assert_not_called() + stdin.assert_not_called() + + +def test_prerecorded_passes_timeout_to_the_client(): + client = Mock() + client.transcribe_file.return_value = {"results": {"channels": []}} + _prerecorded(client, timeout_s=42.0) + assert client.transcribe_file.call_args.kwargs == { + "timeout_s": 42.0, + "file_id": None, + } + + +def test_help_shows_timeout(): + result = _invoke(["--help"]) + assert "--timeout" in result.output + # The approved text, whatever the wrap width. + assert ( + "Seconds to wait at each step (connecting, sending, the response) " + "before giving up. Not a limit on the whole run. Default: 300. " + "Files and URLs only." + ) in " ".join(result.output.split()) diff --git a/packages/deepctl-core/src/deepctl_core/client.py b/packages/deepctl-core/src/deepctl_core/client.py index bbfb0d4a..ba86112e 100644 --- a/packages/deepctl-core/src/deepctl_core/client.py +++ b/packages/deepctl-core/src/deepctl_core/client.py @@ -203,10 +203,12 @@ def transcribe_file( file_path: str | Path, options: dict[str, Any] | None = None, *, + timeout_s: float | None = None, file_id: tuple[int, int] | None = None, ) -> dict[str, Any]: """Transcribe an audio file. + ``timeout_s`` replaces the client's 60 s timeout for this request. ``file_id`` is ``(st_dev, st_ino)`` of the file the caller validated; the upload stops if the path now names another file. """ @@ -222,30 +224,41 @@ def transcribe_file( if options: default_options.update(options) + if TYPE_CHECKING: + from deepgram.core.request_options import RequestOptions + from .upload import FileBody, UploadFileError try: # Streamed from disk; the SDK resends the same body on a retry. with FileBody(file_path, file_id=file_id) as body: + request_options: RequestOptions = { + "additional_headers": {"Content-Length": str(len(body))} + } + if timeout_s is not None: + request_options["timeout"] = timeout_s response = self.client.listen.v1.media.transcribe_file( - request=body, - request_options={ - "additional_headers": {"Content-Length": str(len(body))} - }, - **default_options, + request=body, request_options=request_options, **default_options ) return cast("dict[str, Any]", response.model_dump()) except UploadFileError: raise except Exception as e: - transport.raise_for(e, self._secrets) + transport.raise_for(e, self._secrets, timeout_s=timeout_s) raise self._failed("Transcription failed", e) def transcribe_url( - self, url: str, options: dict[str, Any] | None = None + self, + url: str, + options: dict[str, Any] | None = None, + *, + timeout_s: float | None = None, ) -> dict[str, Any]: - """Transcribe audio from URL.""" + """Transcribe audio from URL. + + ``timeout_s`` replaces the client's 60 s timeout for this request. + """ default_options: dict[str, Any] = { "model": "nova-3", "smart_format": True, @@ -254,14 +267,20 @@ def transcribe_url( if options: default_options.update(options) + if TYPE_CHECKING: + from deepgram.core.request_options import RequestOptions + + request_options: RequestOptions = {} + if timeout_s is not None: + request_options["timeout"] = timeout_s try: response = self.client.listen.v1.media.transcribe_url( - url=url, **default_options + url=url, request_options=request_options, **default_options ) return cast("dict[str, Any]", response.model_dump()) except Exception as e: - transport.raise_for(e, self._secrets) + transport.raise_for(e, self._secrets, timeout_s=timeout_s) raise self._failed("Transcription failed", e) # ── Text-to-Speech ───────────────────────────────────────────── diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py index 70717dc5..bb1030c5 100644 --- a/packages/deepctl-core/src/deepctl_core/transport.py +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -372,9 +372,15 @@ def _api_message(body: object, status: int, secrets: tuple[str, ...]) -> str: def map_exception( - exc: BaseException, secrets: tuple[str, ...] = () + exc: BaseException, + secrets: tuple[str, ...] = (), + *, + timeout_s: float | None = None, ) -> TransportError | None: - """The coded error for a failed request, or ``None`` when ``exc`` isn't one.""" + """The coded error for a failed request, or ``None`` when ``exc`` isn't one. + + ``timeout_s`` is the request's timeout when it isn't the default. + """ if isinstance(exc, TransportError): return exc if "httpx" in sys.modules: @@ -382,10 +388,11 @@ def map_exception( if isinstance(exc, httpx.TimeoutException): host = _host(exc) + seconds = _seconds(timeout_s or DEFAULT_TIMEOUT_S) return TransportError( "timeout", - {"host": host, "timeout_s": _seconds(DEFAULT_TIMEOUT_S)}, - {"host": host, "timeout_s": _seconds(DEFAULT_TIMEOUT_S)}, + {"host": host, "timeout_s": seconds}, + {"host": host, "timeout_s": seconds}, ) if isinstance(exc, httpx.ProxyError): host = _host(exc) @@ -461,9 +468,14 @@ def _seconds(value: float) -> int | float: return int(value) if float(value).is_integer() else value -def raise_for(exc: BaseException, secrets: tuple[str, ...] = ()) -> None: +def raise_for( + exc: BaseException, + secrets: tuple[str, ...] = (), + *, + timeout_s: float | None = None, +) -> None: """Raise the coded :class:`TransportError` for ``exc``, if it maps to one.""" - mapped = map_exception(exc, secrets) + mapped = map_exception(exc, secrets, timeout_s=timeout_s) if mapped is not None and mapped is not exc: raise mapped from exc if mapped is not None: diff --git a/packages/deepctl-core/tests/unit/test_upload.py b/packages/deepctl-core/tests/unit/test_upload.py index 821ec303..9c401701 100644 --- a/packages/deepctl-core/tests/unit/test_upload.py +++ b/packages/deepctl-core/tests/unit/test_upload.py @@ -65,8 +65,13 @@ def record(request: httpx.Request) -> httpx.Response: seen.append((request, request.read())) return handler(request) + # The same timeout and redirect settings as transport.shared_client(). transport._set_client( - httpx.Client(transport=httpx.MockTransport(record), follow_redirects=True) + httpx.Client( + transport=httpx.MockTransport(record), + timeout=transport.DEFAULT_TIMEOUT_S, + follow_redirects=True, + ) ) config = Config(config_path=str(Path.cwd() / "none.yaml")) return DeepgramClient(config, AuthManager(config)), seen @@ -357,3 +362,47 @@ def test_no_module_level_import_cost(): [sys.executable, "-c", code], capture_output=True, text=True, check=True ) assert out.stdout.strip() == "[]" + + +# ── --timeout ──────────────────────────────────────────────────────── + + +def _timed_out(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("timed out", request=request) + + +@pytest.mark.parametrize( + ("timeout_s", "reported"), [(None, 60), (2.0, 2), (2.5, 2.5), (300.0, 300)] +) +def test_timeout_reports_the_effective_value(tmp_path, timeout_s, reported): + path = _audio(tmp_path, 10) + client, seen = facade(_timed_out) + with pytest.raises(transport.TransportError) as err: + client.transcribe_file(path, timeout_s=timeout_s) + assert err.value.code == "timeout" + assert err.value.details == {"host": "api.deepgram.com", "timeout_s": reported} + assert err.value.error == f"api.deepgram.com didn't respond within {reported} s." + ((request, _),) = seen + assert request.extensions["timeout"]["read"] == (timeout_s or 60) + + +def test_url_timeout_reports_the_effective_value(): + client, seen = facade(_timed_out) + with pytest.raises(transport.TransportError) as err: + client.transcribe_url("https://example.com/a.wav", timeout_s=120) + assert err.value.details["timeout_s"] == 120 + ((request, _),) = seen + assert request.extensions["timeout"] == { + "connect": 120, + "read": 120, + "write": 120, + "pool": 120, + } + + +def test_other_requests_keep_the_60_s_timeout(): + client, seen = facade(_timed_out) + with pytest.raises(transport.TransportError) as err: + client.list_models() + assert err.value.details["timeout_s"] == 60 + assert seen[0][0].extensions["timeout"]["read"] == 60 diff --git a/src/deepctl/_manifest.d/listen.json b/src/deepctl/_manifest.d/listen.json index 53dd5120..d842cb90 100644 --- a/src/deepctl/_manifest.d/listen.json +++ b/src/deepctl/_manifest.d/listen.json @@ -58,6 +58,32 @@ "hidden": false, "help": "Output WebVTT captions instead of plain transcript. In live mode, entries are printed as they arrive. Mutually exclusive with --srt." }, + { + "class": "option", + "name": "timeout", + "opts": [ + "--timeout" + ], + "secondary_opts": [], + "type": { + "name": "float range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": false, + "count": false, + "hidden": false, + "help": "Seconds to wait at each step (connecting, sending, the response) before giving up. Not a limit on the whole run. Default: 300. Files and URLs only." + }, { "class": "option", "name": "no_validate", diff --git a/src/deepctl/_manifest.d/transcribe.json b/src/deepctl/_manifest.d/transcribe.json index 04fc32a6..b2d2fbd3 100644 --- a/src/deepctl/_manifest.d/transcribe.json +++ b/src/deepctl/_manifest.d/transcribe.json @@ -58,6 +58,32 @@ "hidden": false, "help": "Output WebVTT captions instead of plain transcript. In live mode, entries are printed as they arrive. Mutually exclusive with --srt." }, + { + "class": "option", + "name": "timeout", + "opts": [ + "--timeout" + ], + "secondary_opts": [], + "type": { + "name": "float range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": false, + "count": false, + "hidden": false, + "help": "Seconds to wait at each step (connecting, sending, the response) before giving up. Not a limit on the whole run. Default: 300. Files and URLs only." + }, { "class": "option", "name": "no_validate", From 7c39a56d47476a94e1787c3281b97bd7d478cc5a Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:32:18 -0400 Subject: [PATCH 15/18] perf(listen): check URL sources without fetching them Before every URL transcription, dg sent its own HEAD request (and a ranged GET when HEAD failed) to the URL, then printed the whole URL, query string included, on stderr and in the result. A presigned URL carries its token in the query string. - A URL source is checked by format only: an http or https scheme and a host, with no network access. --check-url opts in to the HEAD check. It is a usage error with a file or live source, or with --no-validate. - The --check-url request carries no API key and keeps no cookies, including across its own redirects. Its GET fallback is streamed and reads no body, so a server that ignores Range can't fill memory. - The "Transcribing" line, the result table and URL errors show scheme://host/.../, without user info, query string or fragment (validation.display_url): some hosts put a token in the path. The result's source_path keeps the path and drops the rest (validation.output_url). The request to Deepgram still gets the full URL. A URL passed where a file path was expected is shown the same way in "File not found" errors. - A URL whose user info can't be told apart safely (a password with an unescaped /, ? or #, an empty host, a bad port, or an @ outside the parsed host) is shown as scheme:///... everywhere, the result's source_path included. Error texts drop the user info and query of the URL they quote. - The scheme match is case-insensitive: HTTPS://... is a URL source. - httpx loads only when --check-url runs. - The listen README describes URL sources, --check-url and what source_path keeps. Changelog: **`dg listen `**: in `-o json`/`yaml` output, `source_path` is the URL without its user info, query string or fragment; the request to Deepgram still uses the full URL. URL sources are checked by format only; pass `--check-url` to check that the URL responds (a HEAD request that carries no API key or cookies). --- packages/deepctl-cmd-listen/README.extra.md | 5 + packages/deepctl-cmd-listen/README.md | 6 + .../src/deepctl_cmd_listen/command.py | 74 +++- .../tests/unit/test_prerecorded_upload.py | 296 +++++++++++++- .../src/deepctl_shared_utils/validation.py | 345 ++++++++++------ .../tests/unit/test_validation_url.py | 367 ++++++++++++++++++ src/deepctl/_manifest.d/listen.json | 21 + src/deepctl/_manifest.d/transcribe.json | 21 + 8 files changed, 995 insertions(+), 140 deletions(-) create mode 100644 packages/deepctl-cmd-listen/README.extra.md create mode 100644 packages/deepctl-shared-utils/tests/unit/test_validation_url.py diff --git a/packages/deepctl-cmd-listen/README.extra.md b/packages/deepctl-cmd-listen/README.extra.md new file mode 100644 index 00000000..d42da3eb --- /dev/null +++ b/packages/deepctl-cmd-listen/README.extra.md @@ -0,0 +1,5 @@ +## URL sources + +`dg listen ` checks a URL by format only (an http or https scheme and a host). Pass `--check-url` to check that the URL responds first: a HEAD request, then a 1-byte GET if needed. That check sends no Deepgram API key and keeps no cookies, but any `user:pass@` you put in the URL goes with it, as with any fetch. + +Status lines and errors show the URL as `scheme://host/…/`, without user info, query string or fragment. In `-o json` and `-o yaml` output, `source_path` keeps the full path (automation needs it to identify the file) but drops the user info, query string and fragment. If your host puts a token in the URL path, treat `source_path` as sensitive. The request to Deepgram always uses the full URL. diff --git a/packages/deepctl-cmd-listen/README.md b/packages/deepctl-cmd-listen/README.md index 297ae958..864fa2e9 100644 --- a/packages/deepctl-cmd-listen/README.md +++ b/packages/deepctl-cmd-listen/README.md @@ -31,6 +31,12 @@ pipx run deepctl --help |---------|-------------| | `deepctl listen` | `deepctl_cmd_listen.command:ListenCommand` | +## URL sources + +`dg listen ` checks a URL by format only (an http or https scheme and a host). Pass `--check-url` to check that the URL responds first: a HEAD request, then a 1-byte GET if needed. That check sends no Deepgram API key and keeps no cookies, but any `user:pass@` you put in the URL goes with it, as with any fetch. + +Status lines and errors show the URL as `scheme://host/…/`, without user info, query string or fragment. In `-o json` and `-o yaml` output, `source_path` keeps the full path (automation needs it to identify the file) but drops the user info, query string and fragment. If your host puts a token in the URL path, treat `source_path` as sensitive. The request to Deepgram always uses the full URL. + ## Dependencies - `click>=8.2.0,<9` diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 7ac3ecad..b36e567f 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -85,7 +85,8 @@ def _is_url(s: str) -> bool: - return s.startswith(("http://", "https://")) + # Schemes are case-insensitive: HTTPS://... is a URL, not a file path. + return s[:8].lower().startswith(("http://", "https://")) def _ws_base(client: DeepgramClient) -> str: @@ -365,6 +366,16 @@ def get_arguments(self) -> list[dict[str, Any]]: "is_flag": True, "is_option": True, }, + { + "names": ["--check-url"], + "help": ( + "Check that the URL responds before transcribing: a HEAD " + "request, then a 1-byte GET if needed. Sends no Deepgram " + "API key." + ), + "is_flag": True, + "is_option": True, + }, { "names": ["--timeout"], "help": ( @@ -511,6 +522,11 @@ def handle( save_to = kwargs.get("save_to") probe = kwargs.get("probe", False) no_validate = kwargs.get("no_validate", False) + check_url = kwargs.get("check_url", False) + if check_url and mode != "prerecorded_url": + raise click.UsageError("--check-url works only with a URL source.") + if check_url and no_validate: + raise click.UsageError("--check-url can't be used with --no-validate.") timeout: float | None = kwargs.get("timeout") if timeout is not None and mode in ("stream_mic", "stream_stdin"): raise click.UsageError( @@ -612,6 +628,7 @@ def handle( caption_format=caption_format, config=config, timeout_s=timeout or _PRERECORDED_TIMEOUT_S, + check_url=check_url, ) elif mode == "stream_mic": return self._stream_mic( @@ -721,15 +738,27 @@ def _prerecorded( caption_format: str | None, config: Config, timeout_s: float = _PRERECORDED_TIMEOUT_S, + check_url: bool = False, ) -> BaseResult | CodedErrorResult: from deepctl_shared_utils import ( probe_file, require_ffprobe, - validate_audio_file, - validate_url, + ) + from deepctl_shared_utils.validation import ( + audio_file_problem, + display_url, + output_url, + scrub_url_text, + url_problem, ) source_kind = "url" if is_url else "file" + # A presigned URL carries its token in the query string, and some + # hosts put one in the path: never show it, nor one passed where a + # file path was expected. Status lines and errors show the host and + # the last path segment; the result's source_path keeps the path. + url_like = is_url or "://" in source + shown = display_url(source) if url_like else source # The file validated (and probed) must be the file uploaded: its # identity now, compared again once the upload opens it. @@ -745,21 +774,18 @@ def _prerecorded( # ── Validation ───────────────────────────────────────────────── if not no_validate: + # One line per failure, worded here; warnings as WARN lines. if is_url: - if not validate_url(source, check_accessibility=True): - return BaseResult( - status="error", message=f"URL is not accessible: {source}" - ) + # The format only, unless --check-url asks for a HEAD request. + problem, warnings = url_problem(source, check_accessibility=check_url) + elif not Path(source).exists(): + problem, warnings = f"File not found: {shown}", [] else: - if not Path(source).exists(): - return BaseResult( - status="error", message=f"File not found: {source}" - ) - if not validate_audio_file(source): - return BaseResult( - status="error", - message=f"File does not appear to be audio: {source}", - ) + problem, warnings = audio_file_problem(source) + for warning in warnings: + print_warning(escape(warning), stderr=True) + if problem is not None: + return BaseResult(status="error", message=problem) # ── Optional ffprobe analysis ────────────────────────────────── probe_info: dict[str, Any] | None = None @@ -804,7 +830,7 @@ def _prerecorded( options["numerals"] = "true" # ── Call API ─────────────────────────────────────────────────── - status.print(f"[dim]Transcribing[/dim] {source}") + status.print(f"[dim]Transcribing[/dim] {escape(shown)}") status.print(f"[dim]Model:[/dim] {model} [dim]Language:[/dim] {language}") from deepctl_core.upload import UploadFileError, coded_result @@ -820,11 +846,14 @@ def _prerecorded( ) except UploadFileError as e: return coded_result(e) + except FileNotFoundError: # --no-validate and no such file + raise click.ClickException(f"File not found: {shown}") from None except Exception as e: # Raise (not return an error result) so an API rejection — e.g. a # --redact value the v1 endpoint refuses — exits non-zero and is # visible, rather than printing nothing and exiting 0. - raise click.ClickException(f"Transcription failed: {e}") + detail = scrub_url_text(e, source) if url_like else e + raise click.ClickException(f"Transcription failed: {detail}") # ── Format transcript ────────────────────────────────────────── if diarize: @@ -860,7 +889,7 @@ def _prerecorded( return ListenResult( status="success", source=source_kind, - source_path=source, + source_path=output_url(source) if url_like else source, mode="prerecorded", model=model, language=language, @@ -1827,7 +1856,12 @@ def _print_prerecorded_result(self, result: ListenResult) -> None: table.add_column("Key", style="dim") table.add_column("Value") - table.add_row("Source", result.source_path) + source_shown = result.source_path + if result.source == "url": + from deepctl_shared_utils.validation import display_url + + source_shown = display_url(source_shown) + table.add_row("Source", source_shown) table.add_row("Model", model_name or result.model) if model_version: table.add_row("Version", model_version) diff --git a/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py index 021d7f7a..086ddedd 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_prerecorded_upload.py @@ -1,4 +1,4 @@ -"""``dg listen`` prerecorded uploads (W2.4): streamed file bodies, --timeout.""" +"""``dg listen`` prerecorded sources: streamed files, --timeout, URLs.""" from __future__ import annotations @@ -6,6 +6,7 @@ from typing import Any from unittest.mock import Mock, patch +import click import pytest from click.testing import CliRunner from deepctl_cmd_listen.command import ListenCommand @@ -82,9 +83,11 @@ def test_path_swapped_during_validation_uploads_nothing(tmp_path, monkeypatch): def validate_then_swap(path): os.replace(other, audio) # someone swaps the path mid-validation - return True + return None, [] - monkeypatch.setattr(deepctl_shared_utils, "validate_audio_file", validate_then_swap) + monkeypatch.setattr( + deepctl_shared_utils.validation, "audio_file_problem", validate_then_swap + ) client = Mock(spec=DeepgramClient) client.transcribe_file.side_effect = lambda path, options, **kw: ( DeepgramClient.transcribe_file(client, path, options, **kw) @@ -174,3 +177,290 @@ def test_help_shows_timeout(): "before giving up. Not a limit on the whole run. Default: 300. " "Files and URLs only." ) in " ".join(result.output.split()) + + +# ── URL sources ────────────────────────────────────────────────────── + +SIGNED = "https://bucket.example/call.wav?X-Amz-Signature=URLSENTINELTOKEN" + + +@pytest.mark.parametrize(("check_url", "probed"), [(False, False), (True, True)]) +def test_url_is_checked_by_format_unless_check_url(check_url, probed): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + with patch( + "deepctl_shared_utils.validation.url_problem", return_value=(None, []) + ) as validate: + _prerecorded( + client, SIGNED, is_url=True, no_validate=False, check_url=check_url + ) + validate.assert_called_once_with(SIGNED, check_accessibility=probed) + # The API still gets the whole URL, token included. + assert client.transcribe_url.call_args.args[0] == SIGNED + + +def test_no_validate_skips_both_checks(): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + with patch("deepctl_shared_utils.validation.url_problem") as validate: + _prerecorded(client, SIGNED, is_url=True, no_validate=True, check_url=True) + validate.assert_not_called() + + +def test_url_query_never_reaches_stderr_or_the_result(capsys): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + result = _prerecorded(client, SIGNED, is_url=True, no_validate=False) + err = capsys.readouterr().err + assert "Transcribing https://bucket.example/call.wav" in err + assert "URLSENTINELTOKEN" not in err + assert result.source_path == "https://bucket.example/call.wav" + assert "URLSENTINELTOKEN" not in result.model_dump_json() + + +def _unreachable(monkeypatch): + """--check-url's request fails to connect; nothing leaves the test.""" + import httpx + + def refuse(self, method, url, **kwargs): + raise httpx.ConnectError("connection refused") + + monkeypatch.setattr(httpx.Client, "request", refuse) + monkeypatch.setattr(httpx.Client, "send", refuse) + + +@pytest.mark.parametrize( + ("url", "check_url", "message"), + [ + ( + SIGNED.replace("https:", "ftp:"), + False, + "URLs must start with http:// or https://: ftp://bucket.example/call.wav", + ), + ( + SIGNED, + True, + "--check-url couldn't reach https://bucket.example/call.wav " + "(connection refused)", + ), + ], +) +def test_failed_url_check_message_drops_the_query( + url, check_url, message, monkeypatch, capsys +): + _unreachable(monkeypatch) + result = _prerecorded( + Mock(), url, is_url=True, no_validate=False, check_url=check_url + ) + assert result.status == "error" + assert result.message == message + # One report of the failure: the result, nothing extra on stderr. + out = capsys.readouterr() + assert "reach" not in out.err and "http://" not in out.err + assert "URLSENTINELTOKEN" not in out.err + out.out + + +@pytest.mark.parametrize(("args", "expected"), [([], False), (["--check-url"], True)]) +def test_check_url_flag_reaches_prerecorded(args, expected): + done = ListenResult(status="success", source="url", mode="prerecorded") + prerecorded = Mock(return_value=done) + result = _invoke([SIGNED, *args], prerecorded) + assert result.exit_code == 0, result.output + assert prerecorded.call_args.kwargs["check_url"] is expected + + +def test_help_shows_check_url(): + output = _invoke(["--help"]).output + assert "--check-url" in output + # The approved text, whatever the wrap width. + assert ( + "Check that the URL responds before transcribing: a HEAD request, " + "then a 1-byte GET if needed. Sends no Deepgram API key." + ) in " ".join(output.split()) + + +# ── Scheme case, URL-looking paths, and --check-url ── + +UPPER = "HTTPS://u:URLPASS@bucket.example/call.wav?X-Amz-Signature=URLSENTINELTOKEN" + + +@pytest.mark.parametrize("url", [UPPER, "Http://h.example/a.wav", "hTtPs://h/a"]) +def test_scheme_is_case_insensitive(url): + from deepctl_cmd_listen.command import _is_url + + assert _is_url(url) + + +def test_uppercase_scheme_routes_to_the_url_path(): + done = ListenResult(status="success", source="url", mode="prerecorded") + prerecorded = Mock(return_value=done) + result = _invoke([UPPER], prerecorded) + assert result.exit_code == 0, result.output + assert prerecorded.call_args.kwargs["is_url"] is True + + +def test_uppercase_url_never_shows_the_token(capsys): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + result = _prerecorded(client, UPPER, is_url=True, no_validate=False) + err = capsys.readouterr().err + assert "URLSENTINELTOKEN" not in err and "URLPASS" not in err + assert result.source_path == "https://bucket.example/call.wav" + + +URL_AS_PATH = "ftp://u:URLPASS@h.example/a.wav?X-Amz-Signature=URLSENTINELTOKEN" + + +def test_url_looking_path_not_found_hides_the_token(capsys): + result = _prerecorded(Mock(), URL_AS_PATH, no_validate=False) + assert result.status == "error" + assert result.message == "File not found: ftp://h.example/a.wav" + assert "URLSENTINELTOKEN" not in capsys.readouterr().err + + +def test_url_looking_path_not_found_without_validation_hides_the_token(capsys): + from deepctl_core.client import DeepgramClient + + client = Mock(spec=DeepgramClient) + client.transcribe_file.side_effect = lambda path, options, **kw: ( + DeepgramClient.transcribe_file(client, path, options, **kw) + ) + with pytest.raises(click.ClickException) as err: + _prerecorded(client, URL_AS_PATH, no_validate=True) + assert err.value.message == ( + "File not found: ftp://h.example/a.wav" + ) + assert "URLSENTINELTOKEN" not in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["a.wav", "--check-url"], "--check-url works only with a URL source."), + ( + [SIGNED, "--check-url", "--no-validate"], + "--check-url can't be used with --no-validate.", + ), + ( + ["a.wav", "--check-url", "--no-validate"], + "--check-url works only with a URL source.", + ), + (["--mic", "--check-url"], "--check-url works only with a URL source."), + (["-", "--check-url"], "--check-url works only with a URL source."), + ], +) +def test_check_url_without_a_checked_url_is_a_usage_error(args, message): + prerecorded, mic, stdin = Mock(), Mock(), Mock() + result = _invoke(args, prerecorded, mic=mic, stdin=stdin) + assert result.exit_code == 2 + assert message in result.output + prerecorded.assert_not_called() + mic.assert_not_called() + stdin.assert_not_called() + assert "URLSENTINELTOKEN" not in result.output + + +# ── A token in the URL path ── + +DEEP = ( + "https://u:URLPASS@cdn.example/s/URLPATHTOKEN/calls/call.wav?sig=URLSENTINELTOKEN" +) + + +def test_deep_url_path_is_cut_in_status_lines_but_kept_in_the_result(capsys): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + result = _prerecorded(client, DEEP, is_url=True, no_validate=False) + err = capsys.readouterr().err + assert "Transcribing https://cdn.example/…/call.wav" in err + assert "URLPATHTOKEN" not in err and "URLSENTINELTOKEN" not in err + assert "URLPASS" not in err + # Machine output keeps the path, never the user info or query string. + assert result.source_path == "https://cdn.example/s/URLPATHTOKEN/calls/call.wav" + assert "URLSENTINELTOKEN" not in result.model_dump_json() + # The request to Deepgram still gets the full URL. + assert client.transcribe_url.call_args.args[0] == DEEP + + +@pytest.mark.parametrize("check_url", [False, True]) +def test_deep_url_error_text_shows_only_the_last_segment(check_url, monkeypatch): + _unreachable(monkeypatch) + url = DEEP if check_url else DEEP.replace("https:", "ftp:") + result = _prerecorded( + Mock(), url, is_url=True, no_validate=False, check_url=check_url + ) + assert "://cdn.example/…/call.wav" in result.message + assert "URLPATHTOKEN" not in result.message + + +def test_deep_url_table_shows_only_the_last_segment(capsys): + result = ListenResult( + status="success", + source="url", + source_path="https://cdn.example/s/URLPATHTOKEN/calls/call.wav", + mode="prerecorded", + model="nova-3", + ) + ListenCommand()._print_prerecorded_result(result) + out = capsys.readouterr().out + assert "https://cdn.example/…/call.wav" in out + assert "URLPATHTOKEN" not in out + + +# ── User info that urlparse can't see ── + +MALFORMED = [ + "https://u:pa#URLPW@cdn.example/a.wav", + "https://u:pa?URLPW@cdn.example/a.wav", + "https://u:pa/URLPW@cdn.example/a/b.wav", + "https:///URLPW@cdn.example/y/a.wav", + "https://u/URLPW@cdn.example/a.wav", +] + + +@pytest.mark.parametrize("url", MALFORMED) +@pytest.mark.parametrize("check_url", [False, True]) +def test_malformed_user_info_never_reaches_messages(url, check_url, capsys): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + # No request leaves the test: the HEAD check itself reports failure. + with patch( + "deepctl_shared_utils.validation._url_reach_problem", + return_value=("--check-url couldn't reach https:///… (HTTP 404)", []), + ): + result = _prerecorded( + client, url, is_url=True, no_validate=False, check_url=check_url + ) + err = capsys.readouterr().err + assert "URLPW" not in err and "u:pa" not in err + if result.status == "error": + assert result.message in ( + "Not a valid URL: https:///…", + "--check-url couldn't reach https:///… (HTTP 404)", + ) + assert "Error" not in err # reported once, by the result + else: + assert result.source_path == "https:///…" + assert "URLPW" not in result.model_dump_json() + # The request to Deepgram still gets the full URL. + assert client.transcribe_url.call_args.args[0] == url + + +@pytest.mark.parametrize("url", MALFORMED) +def test_malformed_user_info_without_validation(url, capsys): + client = Mock() + client.transcribe_url.return_value = {"results": {"channels": []}} + result = _prerecorded(client, url, is_url=True, no_validate=True) + assert "URLPW" not in capsys.readouterr().err + assert result.source_path == "https:///…" + assert client.transcribe_url.call_args.args[0] == url + + +def test_transcription_error_text_drops_user_info(capsys): + url = "https://u:URLPW@cdn.example/a.wav?sig=URLSENTINELTOKEN" + client = Mock() + client.transcribe_url.side_effect = RuntimeError(f"rejected {url}") + with pytest.raises(click.ClickException) as err: + _prerecorded(client, url, is_url=True, no_validate=True) + assert "URLPW" not in err.value.message + assert "URLSENTINELTOKEN" not in err.value.message diff --git a/packages/deepctl-shared-utils/src/deepctl_shared_utils/validation.py b/packages/deepctl-shared-utils/src/deepctl_shared_utils/validation.py index e0284f99..932c5e93 100644 --- a/packages/deepctl-shared-utils/src/deepctl_shared_utils/validation.py +++ b/packages/deepctl-shared-utils/src/deepctl_shared_utils/validation.py @@ -3,7 +3,8 @@ import os import re from pathlib import Path -from urllib.parse import urlparse +from typing import TypeAlias +from urllib.parse import ParseResult, urlparse from .diagnostics import create_diagnostic_console from .models import FileInfo @@ -54,177 +55,287 @@ } -def validate_audio_file(file_path: str | Path) -> bool: - """Validate that a file exists and appears to be an audio file. +# A problem (the error to show, or None) and the warnings to show with it. +Problems: TypeAlias = "tuple[str | None, list[str]]" - Args: - file_path: Path to the file - Returns: - True if file is valid audio file, False otherwise - """ +def _supported() -> str: + return ", ".join(sorted(SUPPORTED_AUDIO_EXTENSIONS)) + + +def audio_file_problem(file_path: str | Path) -> Problems: + """Why ``file_path`` can't be sent as audio (None when it can), and any + warnings. Prints nothing: the caller reports both, once.""" + # A URL passed as a path may carry a token: show it without one. + shown = display_url(str(file_path)) if "://" in str(file_path) else file_path + warnings: list[str] = [] try: path = Path(file_path) - - # Check if file exists if not path.exists(): - console.print(f"[red]Error:[/red] File not found: {file_path}") - return False - - # Check if it's a file (not directory) + return f"File not found: {shown}", warnings if not path.is_file(): - console.print(f"[red]Error:[/red] Path is not a file: {file_path}") - return False - - # Check file extension + return f"Not a file: {shown}", warnings extension = path.suffix.lower() if extension not in SUPPORTED_AUDIO_EXTENSIONS: - console.print( - f"[yellow]Warning:[/yellow] File extension '{extension}' " - f"is not in supported list" + return ( + f"{shown} doesn't look like an audio file " + f"({extension or 'no extension'}). Supported: {_supported()}. " + "Pass --no-validate to send it anyway.", + warnings, ) - console.print( - f"[dim]Supported extensions: " - f"{', '.join(sorted(SUPPORTED_AUDIO_EXTENSIONS))}[/dim]" - ) - return False - - # Check if file is readable if not os.access(path, os.R_OK): - console.print(f"[red]Error:[/red] File is not readable: {file_path}") - return False - - # Check file size (warn if very large) + return f"File is not readable: {shown}", warnings size_mb = path.stat().st_size / (1024 * 1024) if size_mb > 500: # 500 MB - console.print( - f"[yellow]Warning:[/yellow] Large file detected ({size_mb:.1f} MB)" + warnings.append( + f"Large file ({size_mb:.1f} MB): it may take longer to process." ) - console.print("[dim]Large files may take longer to process[/dim]") + return None, warnings + except Exception as e: + return ( + f"Couldn't check {shown}: {scrub_url_text(e, str(file_path))}", + warnings, + ) - return True - except Exception as e: - console.print(f"[red]Error validating file:[/red] {e}") - return False +def _print(problems: Problems) -> bool: + problem, warnings = problems + for warning in warnings: + console.print(f"[yellow]Warning:[/yellow] {warning}") + if problem is not None: + console.print(f"[red]Error:[/red] {problem}") + return problem is None -def validate_url(url: str, check_accessibility: bool = True) -> bool: - """Validate URL format and optionally check accessibility. +def validate_audio_file(file_path: str | Path) -> bool: + """Validate that a file exists and appears to be an audio file. + + Prints the problem and warnings (see :func:`audio_file_problem`). Args: - url: URL to validate - check_accessibility: Whether to check if URL is accessible + file_path: Path to the file Returns: - True if URL is valid, False otherwise + True if file is valid audio file, False otherwise + """ + return _print(audio_file_problem(file_path)) + + +_SCHEME_RE = re.compile(r"\s*([A-Za-z][A-Za-z0-9+.\-]*)://(.*)", re.S) + + +def _parsed(url: str) -> ParseResult | None: + """``urlparse(url)`` when it has a scheme, a host and a valid port. + + ``None`` otherwise: a password holding an unescaped ``/``, ``?`` or + ``#`` ends the authority early and shows up as a bad port or host. """ + if not _SCHEME_RE.match(url): + return None try: - # Parse URL - parsed = urlparse(url) + parts = urlparse(url) + parts.port # noqa: B018 (raises ValueError on a bad port) + except ValueError: + return None + if not parts.scheme or not parts.hostname: + return None + return parts - # Check basic URL structure - if not parsed.scheme or not parsed.netloc: - console.print(f"[red]Error:[/red] Invalid URL format: {url}") - return False - # Check scheme - if parsed.scheme not in ("http", "https"): - console.print( - f"[red]Error:[/red] Only HTTP/HTTPS URLs are supported: {url}" - ) - return False +def _url_parts(url: str) -> tuple[str, str, str] | None: + """``(scheme, host[:port], path)`` when no user info can show. + + ``None`` when the URL doesn't parse, or when an ``@`` appears outside the + parsed authority: that may be user info whose password holds ``/``, ``?`` + or ``#``, so no part of it is shown. + """ + parts = _parsed(url) + if parts is None: + return None + match = _SCHEME_RE.match(url) + rest = match.group(2) if match else "" + if rest.count("@") != parts.netloc.count("@"): + return None + return parts.scheme, parts.netloc.rpartition("@")[2], parts.path + + +def _redacted(url: str) -> str: + """The shown form of a URL that can't be parsed safely.""" + match = _SCHEME_RE.match(url) + if match: + return f"{match.group(1).lower()}:///…" + if "@" in url: + return "" + return re.split(r"[?#]", url, maxsplit=1)[0] + + +def output_url(url: str) -> str: + """``url`` for machine output: no user info, query string or fragment. + + Presigned URLs carry their token in the query string, so a URL is never + shown, logged or echoed whole. The path is kept: automation needs it to + identify the file. A URL that can't be parsed safely is shown as + ``scheme:///…``. + """ + parts = _url_parts(url) + if parts is None: + return _redacted(url) + scheme, host, path = parts + return f"{scheme}://{host}{path}" + + +def display_url(url: str) -> str: + """``url`` for status lines and errors: ``scheme://host/…/``. - # Check if URL looks like an audio file + As :func:`output_url`, and the path is cut to its last segment, since some + hosts put a token in the path. A path of one segment is shown whole. + """ + parts = _url_parts(url) + if parts is None: + return _redacted(url) + scheme, host, path = parts + segments = [s for s in path.split("/") if s] + if len(segments) <= 1: + return f"{scheme}://{host}{path}" + tail = "/" if path.endswith("/") else "" + return f"{scheme}://{host}/…/{segments[-1]}{tail}" + + +def scrub_url_text(text: object, url: str) -> str: + """``text`` (an error message) with ``url``'s query and user info removed. + + Every piece of the raw text before the URL's last ``@`` is replaced, so a + message that quotes part of a password shows ```` instead. + """ + out = str(text) + match = _SCHEME_RE.match(url) + rest = match.group(2) if match else url + if "@" in rest: + secret = rest.rpartition("@")[0] + pieces = [secret, *re.split(r"[:/?#@]", secret)] + for piece in sorted(pieces, key=len, reverse=True): + if len(piece) >= 2: + out = out.replace(piece, "") + return _no_query(out) + + +def url_problem(url: str, check_accessibility: bool = False) -> Problems: + """Why ``url`` can't be sent (None when it can), and any warnings. + + By default only the format is checked (an http or https scheme and a + host), with no network access. ``check_accessibility`` also sends a + HEAD request (no credentials, no cookies). Prints nothing: the caller + reports both, once. + """ + shown = display_url(url) + warnings: list[str] = [] + try: + parsed = _parsed(url) + if parsed is None: + return f"Not a valid URL: {shown}", warnings + if parsed.scheme not in ("http", "https"): + return f"URLs must start with http:// or https://: {shown}", warnings path = parsed.path.lower() if path: extension = Path(path).suffix if extension and extension not in SUPPORTED_AUDIO_EXTENSIONS: - console.print( - f"[yellow]Warning:[/yellow] URL doesn't appear to be an " - f"audio file: {url}" + warnings.append( + f"{shown} doesn't look like an audio file ({extension}). " + f"Supported: {_supported()}." ) - console.print( - f"[dim]Expected extensions: " - f"{', '.join(sorted(SUPPORTED_AUDIO_EXTENSIONS))}[/dim]" - ) - - # Check accessibility if requested if check_accessibility: - return _check_url_accessibility(url) - - return True - + problem, more = _url_reach_problem(url) + return problem, warnings + more + return None, warnings except Exception as e: - console.print(f"[red]Error validating URL:[/red] {e}") - return False + return f"Not a valid URL: {shown} ({scrub_url_text(e, url)})", warnings -def _check_url_accessibility(url: str) -> bool: - """Check if URL is accessible. +def validate_url(url: str, check_accessibility: bool = False) -> bool: + """Validate URL format and optionally check accessibility. + + Prints the problem and warnings (see :func:`url_problem`). Args: - url: URL to check + url: URL to validate + check_accessibility: Also send a HEAD request (no credentials, no + cookies) to check the URL is reachable Returns: - True if accessible, False otherwise + True if URL is valid, False otherwise + """ + return _print(url_problem(url, check_accessibility)) + + +def _no_query(text: object) -> str: + """``text`` with any URL query string removed.""" + return re.sub(r"\?\S*", "", str(text)) + + +_CHECK_TIMEOUT_S = 10.0 + + +def _url_reach_problem(url: str) -> Problems: + """``--check-url``: why ``url`` can't be reached (None when it can), + and warnings about what it serves. + + The request carries no API key and keeps no cookies: it goes to a host + the user named, not to Deepgram. """ import httpx + shown = display_url(url) + warnings: list[str] = [] + + def unreachable(why: str) -> Problems: + return f"--check-url couldn't reach {shown} ({why})", warnings + try: - console.print("[dim]Checking URL accessibility...[/dim]") + with httpx.Client(timeout=_CHECK_TIMEOUT_S) as client: + from http.cookiejar import DefaultCookiePolicy - with httpx.Client(timeout=10.0) as client: + client.cookies.jar.set_policy(DefaultCookiePolicy(allowed_domains=[])) # Try HEAD first; fall back to a ranged GET if the server # doesn't support HEAD (some CDNs/redirect services return # 404 or 405 for HEAD but serve GET fine). response = client.head(url, follow_redirects=True) if response.status_code not in range(200, 400): - response = client.get( - url, - follow_redirects=True, - headers={"Range": "bytes=0-0"}, + # Streamed, and the body is never read: a server that + # ignores Range can't make this load the whole file. + with client.stream( + "GET", url, follow_redirects=True, headers={"Range": "bytes=0-0"} + ) as response: + pass + + if response.status_code not in (200, 206): + return unreachable(f"HTTP {response.status_code}") + content_type = response.headers.get("content-type", "").lower() + if content_type and not any( + mime in content_type for mime in AUDIO_MIME_TYPES + ): + warnings.append( + f"{shown} is served as {content_type}, which may not be audio." ) - - if response.status_code in (200, 206): - # Check content type if available - content_type = response.headers.get("content-type", "").lower() - if content_type and not any( - mime in content_type for mime in AUDIO_MIME_TYPES - ): - console.print( - f"[yellow]Warning:[/yellow] Content type may not be " - f"audio: {content_type}" + content_length = response.headers.get("content-length") + if content_length and content_length.isdigit(): + size_mb = int(content_length) / (1024 * 1024) + if size_mb > 500: # 500 MB + warnings.append( + f"Large file ({size_mb:.1f} MB): it may take longer to process." ) - - # Check content length if available - content_length = response.headers.get("content-length") - if content_length: - size_mb = int(content_length) / (1024 * 1024) - if size_mb > 500: # 500 MB - console.print( - f"[yellow]Warning:[/yellow] Large file detected " - f"({size_mb:.1f} MB)" - ) - - return True - else: - console.print( - f"[red]Error:[/red] URL not accessible " - f"(HTTP {response.status_code}): {url}" - ) - return False + return None, warnings except httpx.TimeoutException: - console.print(f"[red]Error:[/red] URL request timed out: {url}") - return False - except httpx.RequestError as e: - console.print(f"[red]Error:[/red] Failed to access URL: {e}") - return False + return unreachable(f"no answer within {_CHECK_TIMEOUT_S:g} s") except Exception as e: - console.print(f"[red]Error checking URL accessibility:[/red] {e}") - return False + return unreachable(scrub_url_text(e, url) or type(e).__name__) + + +def _check_url_accessibility(url: str) -> bool: + """Check if URL is accessible (see :func:`_url_reach_problem`); prints + the problem and warnings.""" + return _print(_url_reach_problem(url)) def validate_api_key(api_key: str) -> bool: diff --git a/packages/deepctl-shared-utils/tests/unit/test_validation_url.py b/packages/deepctl-shared-utils/tests/unit/test_validation_url.py new file mode 100644 index 00000000..9c0e0fee --- /dev/null +++ b/packages/deepctl-shared-utils/tests/unit/test_validation_url.py @@ -0,0 +1,367 @@ +"""URL source checks: format only by default, an opt-in HEAD request.""" + +from __future__ import annotations + +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from io import StringIO +from typing import ClassVar + +import httpx +import pytest +from deepctl_shared_utils import validation +from deepctl_shared_utils.validation import ( + display_url, + output_url, + scrub_url_text, + validate_url, +) +from rich.console import Console + +TOKEN = "X-Amz-Signature=URLSENTINELTOKEN" + + +@pytest.fixture +def printed(monkeypatch) -> StringIO: + buffer = StringIO() + monkeypatch.setattr( + validation, "console", Console(file=buffer, width=400, no_color=True) + ) + return buffer + + +@pytest.mark.parametrize( + ("url", "shown"), + [ + ("https://h.example/a.wav", "https://h.example/a.wav"), + (f"https://h.example/a.wav?{TOKEN}&b=1", "https://h.example/a.wav"), + ("https://u:p@h.example:8443/x/a.wav#t", "https://h.example:8443/x/a.wav"), + (f"http://[::1]:8080/a.wav?{TOKEN}", "http://[::1]:8080/a.wav"), + (f"not a url?{TOKEN}", "not a url"), + ], +) +def test_output_url(url, shown): + assert output_url(url) == shown + # One path segment or none: status lines show the same thing. + if shown.count("/") <= 3: + assert display_url(url) == shown + + +@pytest.mark.parametrize( + ("url", "full", "shown"), + [ + ( + f"https://u:p@cdn.example/s/URLPATHTOKEN/a.wav?{TOKEN}", + "https://cdn.example/s/URLPATHTOKEN/a.wav", + "https://cdn.example/…/a.wav", + ), + ( + "https://h.example:8443/x/y/", + "https://h.example:8443/x/y/", + "https://h.example:8443/…/y/", + ), + ( + "https://u:p@h.example:8443/x/a.wav#t", + "https://h.example:8443/x/a.wav", + "https://h.example:8443/…/a.wav", + ), + ("https://h.example/a/", "https://h.example/a/", "https://h.example/a/"), + ("https://h.example", "https://h.example", "https://h.example"), + ("https://h.example/", "https://h.example/", "https://h.example/"), + ], +) +def test_display_url_keeps_only_the_last_path_segment(url, full, shown): + assert output_url(url) == full + assert display_url(url) == shown + + +def test_format_check_makes_no_request(monkeypatch, printed): + def no_client(*args, **kwargs): + raise AssertionError("validate_url opened an HTTP client") + + monkeypatch.setattr(httpx, "Client", no_client) + assert validate_url(f"https://h.example/a.wav?{TOKEN}") is True + assert printed.getvalue() == "" + + +@pytest.mark.parametrize( + "url", + [ + f"https:///a.wav?{TOKEN}", + f"https://user@/a.wav?{TOKEN}", + f"ftp://h.example/a.wav?{TOKEN}", + f"h.example/a.wav?{TOKEN}", + ], +) +def test_bad_format_is_rejected_without_the_query(url, printed): + assert validate_url(url) is False + assert "Error:" in printed.getvalue() + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +def test_non_audio_warning_drops_the_query(printed): + assert validate_url(f"https://h.example/page.html?{TOKEN}") is True + assert "https://h.example/page.html" in printed.getvalue() + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +class _Origin(BaseHTTPRequestHandler): + """Sets a cookie and redirects once, then answers 200 or ``status``.""" + + seen: ClassVar[list[dict[str, str | None]]] = [] + status = 200 + + def do_HEAD(self): + _Origin.seen.append( + { + "path": self.path, + "authorization": self.headers.get("Authorization"), + "cookie": self.headers.get("Cookie"), + } + ) + if self.path.startswith("/start"): + self.send_response(302) + self.send_header("Set-Cookie", "session=URLCOOKIE; Path=/") + self.send_header("Location", "/audio.wav?" + TOKEN) + else: + self.send_response(_Origin.status) + self.send_header("Content-Type", "audio/wav") + self.send_header("Content-Length", "0") + self.end_headers() + + do_GET = do_HEAD + + def log_message(self, *args): + pass + + +@pytest.fixture +def origin(): + _Origin.seen = [] + _Origin.status = 200 + server = ThreadingHTTPServer(("127.0.0.1", 0), _Origin) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_address[1]}" + server.shutdown() + server.server_close() + + +def test_check_url_sends_head_without_credentials_or_cookies(origin, printed): + assert validate_url(f"{origin}/start/a.wav?{TOKEN}", check_accessibility=True) + assert [r["path"].split("?")[0] for r in _Origin.seen] == [ + "/start/a.wav", + "/audio.wav", + ] + assert all(r["authorization"] is None for r in _Origin.seen) + assert all(r["cookie"] is None for r in _Origin.seen) + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +def test_check_url_failure_drops_the_query(origin, printed): + _Origin.status = 403 + assert not validate_url(f"{origin}/a.wav?{TOKEN}", check_accessibility=True) + assert "HTTP 403" in printed.getvalue() + assert f"{origin}/a.wav" in printed.getvalue() + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +def test_check_url_connection_error_drops_the_query(printed): + url = f"http://127.0.0.1:9/a.wav?{TOKEN}" + assert not validate_url(url, check_accessibility=True) + assert "--check-url couldn't reach http://127.0.0.1:9/a.wav (" in printed.getvalue() + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +def test_httpx_loads_only_for_the_check(): + import subprocess + import sys + + code = ( + "import sys; from deepctl_shared_utils import validation; " + "validation.validate_url('https://h.example/a.wav'); " + "print('httpx' in sys.modules)" + ) + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.strip() == "False" + + +# ── The GET fallback never reads the body ──────────────────────────── + +BIG = 100 * 1024 * 1024 + + +class _IgnoresRange(BaseHTTPRequestHandler): + """HEAD -> 405; GET ignores Range and sends 100 MB.""" + + protocol_version = "HTTP/1.1" + sent: ClassVar[list[int]] = [] + + def do_HEAD(self): + self.send_response(405) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "audio/wav") + self.send_header("Content-Length", str(BIG)) + self.end_headers() + chunk = b"\0" * (1 << 16) + sent = 0 + try: + while sent < BIG: + self.wfile.write(chunk) + sent += len(chunk) + except OSError: + pass + _IgnoresRange.sent.append(sent) + + def log_message(self, *args): + pass + + +def test_get_fallback_reads_headers_only(printed): + _IgnoresRange.sent = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _IgnoresRange) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_address[1]}/big.wav?{TOKEN}" + assert validate_url(url, check_accessibility=True) is True + deadline = time.monotonic() + 10 + while not _IgnoresRange.sent and time.monotonic() < deadline: + time.sleep(0.05) + server.shutdown() + finally: + server.server_close() + # The client closed the connection after the headers: the server could + # write only what fits in the socket buffers, far short of 100 MB. + assert _IgnoresRange.sent and _IgnoresRange.sent[0] < BIG // 4 + assert "URLSENTINELTOKEN" not in printed.getvalue() + + +# ── User info that urlparse can't see ── + +REDACTED = "https:///…" + + +@pytest.mark.parametrize( + ("url", "valid_format"), + [ + # A password with an unescaped #, ? or / ends the authority early. + ("https://u:pa#URLPW@cdn.example/a.wav", False), + ("https://u:pa?URLPW@cdn.example/a.wav", False), + ("https://u:pa/URLPW@cdn.example/a/b.wav", False), + (f"https://u:pa/URLPW@cdn.example/a/b.wav?{TOKEN}", False), + # An empty authority: a WHATWG parser reads this as user info. + ("https:///URLPW@cdn.example/y/a.wav", False), + ("https://user@/a.wav", False), + # Parses, but an @ sits outside the authority: never shown. + ("https://u/URLPW@cdn.example/a.wav", True), + ("https://user?URLPW@cdn.example/a.wav", True), + ("HTTPS://u/URLPW@cdn.example/x/a.wav", True), + ], +) +def test_malformed_user_info_is_redacted(url, valid_format, printed): + assert output_url(url) == REDACTED + assert display_url(url) == REDACTED + assert validate_url(url) is valid_format + shown = printed.getvalue() + assert "URLPW" not in shown and "u:pa" not in shown + if not valid_format: + assert f"Not a valid URL: {REDACTED}" in shown + + +@pytest.mark.parametrize( + ("url", "full", "shown"), + [ + ( + "https://u%40x:URLPW%40y@h.example/a/b.wav", + "https://h.example/a/b.wav", + "https://h.example/…/b.wav", + ), + ( + f"https://u:URLPW@[::1]:8443/x/a.wav?{TOKEN}", + "https://[::1]:8443/x/a.wav", + "https://[::1]:8443/…/a.wav", + ), + ( + "https://u:URLPW@h.example:8443/a.wav", + "https://h.example:8443/a.wav", + "https://h.example:8443/a.wav", + ), + ( + "https://u:URLPW@[2001:db8::1]/s/a.wav", + "https://[2001:db8::1]/s/a.wav", + "https://[2001:db8::1]/…/a.wav", + ), + ], +) +def test_well_formed_user_info_is_dropped(url, full, shown, printed): + assert output_url(url) == full + assert display_url(url) == shown + assert validate_url(url) is True + assert "URLPW" not in printed.getvalue() + + +@pytest.mark.parametrize( + "url", ["https://h.example:99999/a.wav", "https://[::1/a.wav", "https://h:x/a"] +) +def test_unparseable_url_is_redacted(url, printed): + assert display_url(url) == REDACTED + assert validate_url(url) is False + assert f"Not a valid URL: {REDACTED}" in printed.getvalue() + + +def test_scrub_url_text_hides_quoted_password_pieces(): + url = f"https://u:pa#URLPW@cdn.example/a.wav?{TOKEN}" + text = scrub_url_text(f"Invalid port: 'pa' in {url}", url) + assert "URLPW" not in text and "'pa'" not in text + assert "URLSENTINELTOKEN" not in text + + +def test_check_url_errors_never_quote_user_info(monkeypatch, printed): + url = "https://u/URLPW@cdn.example/a.wav" + + class _Boom: + def __init__(self, *args, **kwargs): + raise httpx.ConnectError(f"cannot connect to {url}") + + monkeypatch.setattr(httpx, "Client", _Boom) + assert validate_url(url, check_accessibility=True) is False + assert "URLPW" not in printed.getvalue() + + +def test_url_problem_prints_nothing(printed): + from deepctl_shared_utils.validation import url_problem + + assert url_problem("https://h.example:99999/a.wav") == ( + f"Not a valid URL: {REDACTED}", + [], + ) + assert url_problem("ftp://h.example/a.wav")[0] == ( + "URLs must start with http:// or https://: ftp://h.example/a.wav" + ) + problem, warnings = url_problem("https://h.example/a.txt") + assert problem is None + assert warnings and "doesn't look like an audio file (.txt)" in warnings[0] + assert printed.getvalue() == "" + + +def test_audio_file_problem_names_the_way_out(tmp_path, printed): + from deepctl_shared_utils.validation import audio_file_problem + + doc = tmp_path / "notes.xyz" + doc.write_text("x") + problem, _ = audio_file_problem(doc) + assert problem is not None + assert problem.startswith(f"{doc} doesn't look like an audio file (.xyz).") + assert problem.endswith("Pass --no-validate to send it anyway.") + assert audio_file_problem(tmp_path / "gone.wav")[0] == ( + f"File not found: {tmp_path / 'gone.wav'}" + ) + assert printed.getvalue() == "" diff --git a/src/deepctl/_manifest.d/listen.json b/src/deepctl/_manifest.d/listen.json index d842cb90..fce76bd2 100644 --- a/src/deepctl/_manifest.d/listen.json +++ b/src/deepctl/_manifest.d/listen.json @@ -84,6 +84,27 @@ "hidden": false, "help": "Seconds to wait at each step (connecting, sending, the response) before giving up. Not a limit on the whole run. Default: 300. Files and URLs only." }, + { + "class": "option", + "name": "check_url", + "opts": [ + "--check-url" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Check that the URL responds before transcribing: a HEAD request, then a 1-byte GET if needed. Sends no Deepgram API key." + }, { "class": "option", "name": "no_validate", diff --git a/src/deepctl/_manifest.d/transcribe.json b/src/deepctl/_manifest.d/transcribe.json index b2d2fbd3..5a7f1b6d 100644 --- a/src/deepctl/_manifest.d/transcribe.json +++ b/src/deepctl/_manifest.d/transcribe.json @@ -84,6 +84,27 @@ "hidden": false, "help": "Seconds to wait at each step (connecting, sending, the response) before giving up. Not a limit on the whole run. Default: 300. Files and URLs only." }, + { + "class": "option", + "name": "check_url", + "opts": [ + "--check-url" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Check that the URL responds before transcribing: a HEAD request, then a 1-byte GET if needed. Sends no Deepgram API key." + }, { "class": "option", "name": "no_validate", From 3a2af7b0637e2cfb2a6f84f8df1a53e3c8cb0765 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:18:55 -0400 Subject: [PATCH 16/18] fix(debug): send no stored credentials from debug network debug network now makes its diagnostic requests with httpx instead of requests and urllib3, one short-lived client per request. - The probe no longer attaches credentials from ~/.netrc. requests added them for a matching (or default) netrc entry, including on the request made with certificate verification disabled; httpx does not read netrc. No Deepgram key was ever sent by these requests, and none is now. - CA bundles from the environment keep working: SSL_CERT_FILE or SSL_CERT_DIR first (read by httpx), then REQUESTS_CA_BUNDLE, then CURL_CA_BUNDLE, as requests resolved them. verify=False ignores all of them. HTTP(S)_PROXY and NO_PROXY apply as before. - The JSON payload keeps every key and adds two: environment_info.httpx_version, and library ("httpx") on each entry of python_requests_tests. requests_version and urllib3_version are now read from package metadata. --- .../src/deepctl_cmd_debug_network/command.py | 178 +++++++--- .../src/deepctl_cmd_debug_network/models.py | 4 +- .../tests/unit/test_network_command.py | 328 +++++++++++++----- 3 files changed, 366 insertions(+), 144 deletions(-) diff --git a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py index 3edc4ed7..d80b0fb8 100644 --- a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py +++ b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py @@ -6,8 +6,7 @@ import subprocess import sys import time -import warnings -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse from deepctl_core import AuthManager, BaseCommand, Config, DeepgramClient @@ -26,6 +25,11 @@ TLSTestResult, ) +if TYPE_CHECKING: + import ssl + + import httpx + console = Console() # Default Deepgram endpoints to test @@ -35,9 +39,74 @@ } +def _is_ssl_error(exc: BaseException) -> bool: + """True when an httpx error was caused by a TLS failure (the case + requests raised as SSLError).""" + import ssl + + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + if isinstance(current, ssl.SSLError): + return True + seen.add(id(current)) + current = current.__cause__ or current.__context__ + return False + + +def _requests_ca_bundle() -> str | None: + """The CA bundle requests would have used from the environment. + + httpx reads SSL_CERT_FILE and SSL_CERT_DIR itself; when either is set it + wins. Otherwise REQUESTS_CA_BUNDLE, then CURL_CA_BUNDLE, as requests + resolved them (an empty value counts as unset). + """ + if os.environ.get("SSL_CERT_FILE") or os.environ.get("SSL_CERT_DIR"): + return None + return ( + os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("CURL_CA_BUNDLE") or None + ) + + +def _verify_setting(verify: bool) -> "bool | ssl.SSLContext": + """httpx's verify argument: False stays False; True becomes an SSL + context on the requests-style bundle when one is set in the + environment, and otherwise stays True (httpx defaults, which honour + SSL_CERT_FILE / SSL_CERT_DIR).""" + import ssl + + if not verify: + return False + bundle = _requests_ca_bundle() + if bundle is None: + return True + if os.path.isdir(bundle): + return ssl.create_default_context(capath=bundle) + if not os.path.isfile(bundle): + # requests' wording for the same mistake. + raise OSError( + "Could not find a suitable TLS CA certificate bundle, " + f"invalid path: {bundle}" + ) + return ssl.create_default_context(cafile=bundle) + + +def _dist_version(name: str) -> str: + """Installed version of a distribution, read without importing it.""" + from importlib.metadata import PackageNotFoundError, version + + try: + return version(name) + except PackageNotFoundError: + return "not installed" + + class NetworkCommand(BaseCommand): """Debug network connectivity issues with Deepgram services.""" + # Tests swap in an httpx.MockTransport; None means the real network. + _transport: "httpx.BaseTransport | None" = None + name = "network" help = "Debug network connectivity issues with Deepgram services" short_help = "Debug network issues" @@ -105,6 +174,29 @@ def get_arguments(self) -> list[dict[str, Any]]: }, ] + def _http_client( + self, *, timeout: float, verify: bool = True, follow_redirects: bool = False + ) -> "httpx.Client": + """A throwaway client for one diagnostic request. + + Never the shared CLI client: it carries no credentials (httpx does + not read ~/.netrc), and a verify=False client must not outlive its + one test. trust_env applies HTTP(S)_PROXY / NO_PROXY as requests did. + + CA trust, first match wins (verify=False skips all of it): + SSL_CERT_FILE or SSL_CERT_DIR (read by httpx), then + REQUESTS_CA_BUNDLE, then CURL_CA_BUNDLE, then httpx's default bundle. + """ + import httpx + + return httpx.Client( + timeout=timeout, + verify=_verify_setting(verify), + follow_redirects=follow_redirects, + trust_env=True, + transport=self._transport, + ) + def handle( self, config: Config, @@ -169,7 +261,7 @@ def handle( progress.remove_task(task) # 5. Python requests tests - task = progress.add_task("Testing Python requests library...", total=None) + task = progress.add_task("Testing Python HTTP client...", total=None) self._test_python_requests(endpoint, result, timeout) progress.remove_task(task) @@ -208,14 +300,12 @@ def handle( def _collect_environment_info(self, result: NetworkDebugResult) -> None: """Collect environment information.""" - import requests - import urllib3 - result.environment_info = { "python_version": sys.version, "platform": sys.platform, - "requests_version": requests.__version__, - "urllib3_version": getattr(urllib3, "__version__", "unknown"), + "requests_version": _dist_version("requests"), + "urllib3_version": _dist_version("urllib3"), + "httpx_version": _dist_version("httpx"), "proxy_env_vars": { "http_proxy": os.environ.get("http_proxy", "Not set"), "https_proxy": os.environ.get("https_proxy", "Not set"), @@ -265,28 +355,25 @@ def _test_basic_connectivity( self, hostname: str, result: NetworkDebugResult, timeout: int ) -> None: """Test basic HTTPS connectivity.""" - import requests - endpoint_test = EndpointTestResult( name="HTTPS", url=f"https://{hostname}", reachable=False ) try: start_time = time.time() - response = requests.get( - f"https://{hostname}/", timeout=timeout, allow_redirects=False - ) + with self._http_client(timeout=timeout) as http: + response = http.get(f"https://{hostname}/") endpoint_test.response_time_ms = (time.time() - start_time) * 1000 endpoint_test.status_code = response.status_code endpoint_test.reachable = True endpoint_test.ssl_valid = True - except requests.exceptions.SSLError as e: - endpoint_test.error = f"SSL Error: {e!s}" - endpoint_test.ssl_valid = False - result.network_issues_detected = True except Exception as e: - endpoint_test.error = str(e) + if _is_ssl_error(e): + endpoint_test.error = f"SSL Error: {e!s}" + endpoint_test.ssl_valid = False + else: + endpoint_test.error = str(e) result.network_issues_detected = True result.endpoint_results.append(endpoint_test) @@ -413,8 +500,6 @@ def _test_revocation_endpoints( simulate_blocked_crl: bool = False, ) -> None: """Test accessibility of CRL/OCSP/CA issuer endpoints.""" - import requests - all_endpoints = [] for cert in tls_result.certificate_chain: @@ -451,7 +536,8 @@ def _test_revocation_endpoints( else: try: start_time = time.time() - response = requests.head(url, timeout=5, allow_redirects=True) + with self._http_client(timeout=5, follow_redirects=True) as http: + response = http.head(url) endpoint_test.response_time_ms = (time.time() - start_time) * 1000 endpoint_test.status_code = response.status_code endpoint_test.accessible = response.status_code < 400 @@ -465,10 +551,8 @@ def _test_revocation_endpoints( def _test_python_requests( self, hostname: str, result: NetworkDebugResult, timeout: int ) -> None: - """Test connectivity using Python requests library.""" - import requests - import urllib3 - + """Test connectivity with Python's HTTP client (httpx), with and + without certificate verification. Each request gets its own client.""" # Test with SSL verification enabled test_with_verify = PythonRequestsTest( url=f"https://{hostname}/", success=False, ssl_verify_enabled=True @@ -476,21 +560,18 @@ def _test_python_requests( try: start_time = time.time() - response = requests.get( - f"https://{hostname}/", - timeout=timeout, - verify=True, - allow_redirects=False, - ) + with self._http_client(timeout=timeout, verify=True) as http: + response = http.get(f"https://{hostname}/") test_with_verify.response_time_ms = (time.time() - start_time) * 1000 test_with_verify.status_code = response.status_code test_with_verify.success = True - except requests.exceptions.SSLError as e: - test_with_verify.error = f"SSL verification failed: {e!s}" - result.network_issues_detected = True except Exception as e: - test_with_verify.error = str(e) + if _is_ssl_error(e): + test_with_verify.error = f"SSL verification failed: {e!s}" + result.network_issues_detected = True + else: + test_with_verify.error = str(e) result.python_requests_tests.append(test_with_verify) @@ -500,25 +581,17 @@ def _test_python_requests( ) try: - # Disable SSL warnings temporarily - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - + # An isolated, credential-free client that is closed right after + # this one request; nothing else ever sees verify=False. start_time = time.time() - response = requests.get( - f"https://{hostname}/", - timeout=timeout, - verify=False, - allow_redirects=False, - ) + with self._http_client(timeout=timeout, verify=False) as http: + response = http.get(f"https://{hostname}/") test_without_verify.response_time_ms = (time.time() - start_time) * 1000 test_without_verify.status_code = response.status_code test_without_verify.success = True except Exception as e: test_without_verify.error = str(e) - finally: - # Re-enable warnings - warnings.resetwarnings() result.python_requests_tests.append(test_without_verify) @@ -670,8 +743,6 @@ def _test_multi_region_latency( self, result: NetworkDebugResult, timeout: int ) -> None: """Test latency to known Deepgram endpoints.""" - import requests - endpoints = [ ("api.deepgram.com", "Primary API"), ] @@ -704,11 +775,8 @@ def _test_multi_region_latency( # HTTPS connection timing connect_start = time.time() - resp = requests.get( - f"https://{hostname}/", - timeout=timeout, - allow_redirects=False, - ) + with self._http_client(timeout=timeout) as http: + resp = http.get(f"https://{hostname}/") total_ms = (time.time() - connect_start) * 1000 connect_ms = total_ms - dns_ms if total_ms and dns_ms else None tls_ms = connect_ms * 0.6 if connect_ms else None @@ -855,7 +923,7 @@ def _display_results(self, result: NetworkDebugResult, verbose: bool) -> None: # Python Requests Tests if result.python_requests_tests: - console.print("[bold]Python Requests Library Tests:[/bold]") + console.print("[bold]Python HTTP Client Tests (httpx):[/bold]") for test in result.python_requests_tests: verify_status = ( "with SSL verification" diff --git a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/models.py b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/models.py index 5b44787f..8fe048fc 100644 --- a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/models.py +++ b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/models.py @@ -70,9 +70,11 @@ class TLSTestResult(BaseModel): class PythonRequestsTest(BaseModel): - """Result of testing with Python requests library.""" + """Result of one Python HTTP client request (the payload keeps its + requests-era name; ``library`` says which client made it).""" url: str + library: str = "httpx" success: bool status_code: int | None = None response_time_ms: float | None = None diff --git a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py index be4dae1f..667a17dd 100644 --- a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py +++ b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py @@ -2,12 +2,13 @@ import json import socket +import ssl import subprocess from datetime import datetime from unittest.mock import MagicMock, Mock, call, patch +import httpx import pytest -import requests from deepctl_cmd_debug_network import NetworkCommand from deepctl_cmd_debug_network.models import ( CertificateInfo, @@ -28,6 +29,53 @@ def network_command(): return NetworkCommand() +class _MockHTTP: + """Routes every request the command makes to an httpx.MockTransport and + records the requests and the clients that carried them.""" + + def __init__(self, command, monkeypatch): + self.requests: list[httpx.Request] = [] + self.clients: list[dict] = [] + self.status_code = 404 + self.error: Exception | None = None + command._transport = httpx.MockTransport(self._handle) + original = command._http_client + + def recording_client(**kwargs): + client = original(**kwargs) + kwargs = {"verify": True, "follow_redirects": False, **kwargs} + self.clients.append({"kwargs": kwargs, "client": client}) + return client + + monkeypatch.setattr(command, "_http_client", recording_client) + + def _handle(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if self.error is not None: + raise self.error + return httpx.Response(self.status_code) + + +def _ssl_connect_error() -> httpx.ConnectError: + """What httpx raises for a failed certificate check: a ConnectError + whose cause is the ssl module's verification error.""" + try: + try: + raise ssl.SSLCertVerificationError( + 1, "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed" + ) + except ssl.SSLError as exc: + raise httpx.ConnectError(str(exc)) from exc + except httpx.ConnectError as err: + return err + + +@pytest.fixture +def mock_http(network_command, monkeypatch): + """Every HTTP request goes to a MockTransport; nothing reaches the net.""" + return _MockHTTP(network_command, monkeypatch) + + @pytest.fixture def mock_config(): """Create mock config.""" @@ -69,9 +117,7 @@ def test_get_arguments(self, network_command): assert len(args) == 7 # Check endpoint argument - endpoint_arg = next( - arg for arg in args if "--endpoint" in arg["names"] - ) + endpoint_arg = next(arg for arg in args if "--endpoint" in arg["names"]) assert ( endpoint_arg["help"] == "Specific endpoint to test (default: api.deepgram.com)" @@ -84,13 +130,8 @@ def test_get_arguments(self, network_command): assert verbose_arg["is_flag"] is True # Check skip-commands flag - skip_arg = next( - arg for arg in args if "--skip-commands" in arg["names"] - ) - assert ( - skip_arg["help"] - == "Skip system command execution (openssl, curl, etc.)" - ) + skip_arg = next(arg for arg in args if "--skip-commands" in arg["names"]) + assert skip_arg["help"] == "Skip system command execution (openssl, curl, etc.)" assert skip_arg["is_flag"] is True # Check timeout argument @@ -107,8 +148,7 @@ def test_get_arguments(self, network_command): arg for arg in args if "--simulate-blocked-crl" in arg["names"] ) assert ( - simulate_arg["help"] - == "Simulate blocked CRL/OCSP endpoints (for testing)" + simulate_arg["help"] == "Simulate blocked CRL/OCSP endpoints (for testing)" ) assert simulate_arg["is_flag"] is True @@ -118,9 +158,7 @@ def test_get_arguments(self, network_command): assert save_arg["type"] == str # Check test-websocket flag - ws_arg = next( - arg for arg in args if "--test-websocket" in arg["names"] - ) + ws_arg = next(arg for arg in args if "--test-websocket" in arg["names"]) assert ws_arg["help"] == "Also test WebSocket connectivity (wss://)" assert ws_arg["is_flag"] is True @@ -145,9 +183,7 @@ def test_dns_resolution_success(self, mock_getaddrinfo, network_command): @patch("socket.getaddrinfo") def test_dns_resolution_failure(self, mock_getaddrinfo, network_command): """Test failed DNS resolution.""" - mock_getaddrinfo.side_effect = socket.gaierror( - "Name resolution failed" - ) + mock_getaddrinfo.side_effect = socket.gaierror("Name resolution failed") result = NetworkDebugResult(status="success") network_command._test_dns_resolution("api.deepgram.com", result, 10) @@ -158,17 +194,12 @@ def test_dns_resolution_failure(self, mock_getaddrinfo, network_command): assert dns_result.error == "Name resolution failed" assert result.network_issues_detected is True - @patch("requests.get") - def test_basic_connectivity_success(self, mock_get, network_command): + def test_basic_connectivity_success(self, mock_http, network_command): """Test successful basic HTTPS connectivity.""" - mock_response = Mock() - mock_response.status_code = 404 # Expected for root endpoint - mock_get.return_value = mock_response + mock_http.status_code = 404 # Expected for root endpoint result = NetworkDebugResult(status="success") - network_command._test_basic_connectivity( - "api.deepgram.com", result, 10 - ) + network_command._test_basic_connectivity("api.deepgram.com", result, 10) assert len(result.endpoint_results) == 1 endpoint = result.endpoint_results[0] @@ -178,17 +209,12 @@ def test_basic_connectivity_success(self, mock_get, network_command): assert endpoint.status_code == 404 assert endpoint.ssl_valid is True - @patch("requests.get") - def test_basic_connectivity_ssl_error(self, mock_get, network_command): + def test_basic_connectivity_ssl_error(self, mock_http, network_command): """Test basic connectivity with SSL error.""" - mock_get.side_effect = requests.exceptions.SSLError( - "SSL certificate verification failed" - ) + mock_http.error = _ssl_connect_error() result = NetworkDebugResult(status="success") - network_command._test_basic_connectivity( - "api.deepgram.com", result, 10 - ) + network_command._test_basic_connectivity("api.deepgram.com", result, 10) assert len(result.endpoint_results) == 1 endpoint = result.endpoint_results[0] @@ -236,7 +262,7 @@ def test_analyze_tls_certificates(self, mock_run, network_command): assert tls_result.connected is True assert len(tls_result.certificate_chain) == 2 - def test_test_revocation_endpoints_success(self, network_command): + def test_test_revocation_endpoints_success(self, mock_http, network_command): """Test successful revocation endpoint testing.""" tls_result = TLSTestResult( hostname="api.deepgram.com", port=443, connected=True @@ -252,18 +278,19 @@ def test_test_revocation_endpoints_success(self, network_command): result = NetworkDebugResult(status="success") - with patch("requests.head") as mock_head: - mock_head.return_value = Mock(status_code=200) - network_command._test_revocation_endpoints( - tls_result, result, False - ) + mock_http.status_code = 200 + network_command._test_revocation_endpoints(tls_result, result, False) assert len(tls_result.revocation_endpoints) == 3 for endpoint in tls_result.revocation_endpoints: assert endpoint.accessible is True assert endpoint.status_code == 200 + assert {r.method for r in mock_http.requests} == {"HEAD"} + assert all(c["kwargs"]["follow_redirects"] for c in mock_http.clients) - def test_test_revocation_endpoints_simulated_block(self, network_command): + def test_test_revocation_endpoints_simulated_block( + self, mock_http, network_command + ): """Test revocation endpoints with simulated blocking.""" tls_result = TLSTestResult( hostname="api.deepgram.com", port=443, connected=True @@ -277,6 +304,7 @@ def test_test_revocation_endpoints_simulated_block(self, network_command): ) tls_result.certificate_chain.append(cert) + mock_http.status_code = 200 result = NetworkDebugResult(status="success") network_command._test_revocation_endpoints( tls_result, result, simulate_blocked_crl=True @@ -284,19 +312,13 @@ def test_test_revocation_endpoints_simulated_block(self, network_command): # Check that CRL and OCSP endpoints are marked as blocked ocsp_endpoints = [ - e - for e in tls_result.revocation_endpoints - if e.endpoint_type == "ocsp" + e for e in tls_result.revocation_endpoints if e.endpoint_type == "ocsp" ] crl_endpoints = [ - e - for e in tls_result.revocation_endpoints - if e.endpoint_type == "crl" + e for e in tls_result.revocation_endpoints if e.endpoint_type == "crl" ] ca_endpoints = [ - e - for e in tls_result.revocation_endpoints - if e.endpoint_type == "ca_issuer" + e for e in tls_result.revocation_endpoints if e.endpoint_type == "ca_issuer" ] for endpoint in ocsp_endpoints + crl_endpoints: @@ -304,16 +326,12 @@ def test_test_revocation_endpoints_simulated_block(self, network_command): assert "simulated corporate firewall blocking" in endpoint.error for endpoint in ca_endpoints: - assert ( - endpoint.accessible is True - ) # CA issuer endpoints not blocked + assert endpoint.accessible is True # CA issuer endpoints not blocked - @patch("requests.get") - def test_python_requests_tests(self, mock_get, network_command): - """Test Python requests library connectivity tests.""" - mock_response = Mock() - mock_response.status_code = 404 - mock_get.return_value = mock_response + def test_python_requests_tests(self, mock_http, network_command): + """The Python HTTP client tests run once with and once without + certificate verification, each on its own client.""" + mock_http.status_code = 404 result = NetworkDebugResult(status="success") network_command._test_python_requests("api.deepgram.com", result, 10) @@ -325,19 +343,95 @@ def test_python_requests_tests(self, mock_get, network_command): assert test_with_verify.ssl_verify_enabled is True assert test_with_verify.success is True assert test_with_verify.status_code == 404 + assert test_with_verify.library == "httpx" # Test without SSL verification test_without_verify = result.python_requests_tests[1] assert test_without_verify.ssl_verify_enabled is False assert test_without_verify.success is True assert test_without_verify.status_code == 404 + assert test_without_verify.library == "httpx" + + # Two throwaway clients, the second alone with verify=False, both + # closed after their one request. + assert [c["kwargs"]["verify"] for c in mock_http.clients] == [True, False] + assert len({id(c["client"]) for c in mock_http.clients}) == 2 + assert all(c["client"].is_closed for c in mock_http.clients) + assert [str(r.url) for r in mock_http.requests] == [ + "https://api.deepgram.com/", + "https://api.deepgram.com/", + ] + + def test_python_requests_ssl_error(self, mock_http, network_command): + """A failed certificate check is reported as an SSL failure.""" + mock_http.error = _ssl_connect_error() + + result = NetworkDebugResult(status="success") + network_command._test_python_requests("api.deepgram.com", result, 10) + + with_verify, without_verify = result.python_requests_tests + assert with_verify.success is False + assert with_verify.error.startswith("SSL verification failed: ") + assert "CERTIFICATE_VERIFY_FAILED" in with_verify.error + assert without_verify.success is False + assert result.network_issues_detected is True + + def test_python_requests_timeout(self, mock_http, network_command): + """A timeout is a plain failure, not an SSL one.""" + mock_http.error = httpx.ConnectTimeout("timed out") + + result = NetworkDebugResult(status="success") + network_command._test_python_requests("api.deepgram.com", result, 3) + + with_verify, without_verify = result.python_requests_tests + assert with_verify.success is False + assert with_verify.error == "timed out" + assert without_verify.error == "timed out" + assert result.network_issues_detected is False + assert all(c["kwargs"]["timeout"] == 3 for c in mock_http.clients) + + def test_basic_connectivity_timeout(self, mock_http, network_command): + """A timeout is not an SSL failure, but it is a network issue.""" + mock_http.error = httpx.ReadTimeout("read timed out") + + result = NetworkDebugResult(status="success") + network_command._test_basic_connectivity("api.deepgram.com", result, 10) + + endpoint = result.endpoint_results[0] + assert endpoint.reachable is False + assert endpoint.ssl_valid is True # the model default, as before + assert endpoint.error == "read timed out" + assert result.network_issues_detected is True + + def test_requests_carry_no_credentials( + self, mock_http, network_command, monkeypatch + ): + """No diagnostic request sends an Authorization header, even with a + key in the environment, and none follows a redirect except the + revocation HEADs.""" + monkeypatch.setenv("DEEPGRAM_API_KEY", "sentinel-fake-key-w27") + result = NetworkDebugResult(status="success") + + network_command._test_basic_connectivity("api.deepgram.com", result, 10) + network_command._test_python_requests("api.deepgram.com", result, 10) + + assert len(mock_http.requests) == 3 + for request in mock_http.requests: + assert "authorization" not in request.headers + assert "sentinel-fake-key-w27" not in str(request.headers) + assert not any(c["kwargs"]["follow_redirects"] for c in mock_http.clients) + + def test_http_client_trusts_the_environment(self, network_command): + """The diagnostic client reads proxy/CA env vars like requests did.""" + with network_command._http_client(timeout=5) as client: + assert client.trust_env is True + assert client.follow_redirects is False + assert client.timeout == httpx.Timeout(5) @patch("subprocess.run") def test_run_system_diagnostics(self, mock_run, network_command): """Test system diagnostic command execution.""" - with patch.object( - network_command, "_command_exists", return_value=True - ): + with patch.object(network_command, "_command_exists", return_value=True): mock_run.return_value = Mock( returncode=0, stdout=b"PING api.deepgram.com: 3 packets transmitted, 3 received", @@ -345,9 +439,7 @@ def test_run_system_diagnostics(self, mock_run, network_command): ) result = NetworkDebugResult(status="success") - network_command._run_system_diagnostics( - "api.deepgram.com", result, False - ) + network_command._run_system_diagnostics("api.deepgram.com", result, False) # Should have executed multiple commands assert len(result.command_results) > 0 @@ -359,9 +451,7 @@ def test_run_system_diagnostics(self, mock_run, network_command): @patch("subprocess.run") def test_test_websocket_connectivity(self, mock_run, network_command): """Test WebSocket connectivity testing.""" - with patch.object( - network_command, "_command_exists", return_value=True - ): + with patch.object(network_command, "_command_exists", return_value=True): mock_run.return_value = Mock( returncode=0, stdout=b"HTTP/1.1 401 Unauthorized\r\n", @@ -369,14 +459,10 @@ def test_test_websocket_connectivity(self, mock_run, network_command): ) result = NetworkDebugResult(status="success") - network_command._test_websocket_connectivity( - "api.deepgram.com", result, 10 - ) + network_command._test_websocket_connectivity("api.deepgram.com", result, 10) # Check WebSocket test was added to endpoint results - ws_results = [ - e for e in result.endpoint_results if e.name == "WebSocket" - ] + ws_results = [e for e in result.endpoint_results if e.name == "WebSocket"] assert len(ws_results) == 1 ws_result = ws_results[0] assert ws_result.url == "wss://api.deepgram.com/v1/listen" @@ -460,13 +546,12 @@ def test_generate_recommendations_with_blocked_crl(self, network_command): assert any("r11.c.lencr.org" in r for r in result.recommendations) @patch("socket.getaddrinfo") - @patch("requests.get") @patch("subprocess.run") def test_handle_default_endpoint( self, mock_run, - mock_get, mock_getaddrinfo, + mock_http, network_command, mock_config, mock_auth_manager, @@ -479,7 +564,7 @@ def test_handle_default_endpoint( ] # Mock HTTPS connectivity - mock_get.return_value = Mock(status_code=404) + mock_http.status_code = 404 # Mock subprocess calls mock_run.return_value = Mock(returncode=0, stdout=b"", stderr=b"") @@ -495,13 +580,12 @@ def test_handle_default_endpoint( assert len(result.endpoint_results) > 0 @patch("socket.getaddrinfo") - @patch("requests.get") @patch("subprocess.run") def test_handle_custom_endpoint( self, mock_run, - mock_get, mock_getaddrinfo, + mock_http, network_command, mock_config, mock_auth_manager, @@ -514,7 +598,7 @@ def test_handle_custom_endpoint( ] # Mock HTTPS connectivity - mock_get.return_value = Mock(status_code=200) + mock_http.status_code = 200 # Mock subprocess calls mock_run.return_value = Mock(returncode=0, stdout=b"", stderr=b"") @@ -539,19 +623,87 @@ def test_collect_environment_info(self, network_command): assert "platform" in result.environment_info assert "requests_version" in result.environment_info assert "urllib3_version" in result.environment_info + assert "httpx_version" in result.environment_info + assert result.environment_info["httpx_version"] == httpx.__version__ assert "proxy_env_vars" in result.environment_info def test_collect_environment_info_with_proxy(self, network_command): """Test environment information collection with proxy configured.""" result = NetworkDebugResult(status="success") - with patch.dict( - "os.environ", {"https_proxy": "http://proxy.example.com:8080"} - ): + with patch.dict("os.environ", {"https_proxy": "http://proxy.example.com:8080"}): network_command._collect_environment_info(result) assert result.proxy_detected is True - assert ( - result.proxy_settings["https_proxy"] - == "http://proxy.example.com:8080" - ) + assert result.proxy_settings["https_proxy"] == "http://proxy.example.com:8080" + + +# -- CA bundle environment, as requests resolved it ------------------------ + +_CA_VARS = ("SSL_CERT_FILE", "SSL_CERT_DIR", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE") + + +@pytest.fixture +def clean_ca_env(monkeypatch): + for var in _CA_VARS: + monkeypatch.delenv(var, raising=False) + return monkeypatch + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ({}, None), + ({"REQUESTS_CA_BUNDLE": "/r.pem"}, "/r.pem"), + ({"CURL_CA_BUNDLE": "/c.pem"}, "/c.pem"), + ({"REQUESTS_CA_BUNDLE": "/r.pem", "CURL_CA_BUNDLE": "/c.pem"}, "/r.pem"), + ({"REQUESTS_CA_BUNDLE": "", "CURL_CA_BUNDLE": "/c.pem"}, "/c.pem"), + ({"SSL_CERT_FILE": "/s.pem", "REQUESTS_CA_BUNDLE": "/r.pem"}, None), + ({"SSL_CERT_DIR": "/s", "CURL_CA_BUNDLE": "/c.pem"}, None), + ], +) +def test_requests_ca_bundle_precedence(clean_ca_env, env, expected): + """SSL_CERT_FILE/SSL_CERT_DIR (httpx's own) win; then REQUESTS_CA_BUNDLE, + then CURL_CA_BUNDLE.""" + from deepctl_cmd_debug_network.command import _requests_ca_bundle + + for var, value in env.items(): + clean_ca_env.setenv(var, value) + assert _requests_ca_bundle() == expected + + +def test_verify_setting_uses_the_requests_bundle(clean_ca_env): + import certifi + from deepctl_cmd_debug_network.command import _verify_setting + + assert _verify_setting(True) is True + clean_ca_env.setenv("REQUESTS_CA_BUNDLE", certifi.where()) + context = _verify_setting(True) + assert isinstance(context, ssl.SSLContext) + assert context.verify_mode == ssl.CERT_REQUIRED + assert context.cert_store_stats()["x509_ca"] > 0 + # verify=False is never turned back on by the environment. + assert _verify_setting(False) is False + + +def test_verify_setting_accepts_a_bundle_directory(clean_ca_env, tmp_path): + from deepctl_cmd_debug_network.command import _verify_setting + + clean_ca_env.setenv("CURL_CA_BUNDLE", str(tmp_path)) + assert isinstance(_verify_setting(True), ssl.SSLContext) + + +def test_missing_bundle_is_reported_like_requests( + clean_ca_env, mock_http, network_command, tmp_path +): + """A bad REQUESTS_CA_BUNDLE path fails the verified request with + requests' message; the verify=False request is unaffected.""" + clean_ca_env.setenv("REQUESTS_CA_BUNDLE", str(tmp_path / "missing.pem")) + result = NetworkDebugResult(status="success") + + network_command._test_python_requests("api.deepgram.com", result, 10) + + with_verify, without_verify = result.python_requests_tests + assert with_verify.success is False + assert "Could not find a suitable TLS CA certificate bundle" in with_verify.error + assert without_verify.success is True From b45f74da33ddd178d99706089fcf9600d6108d71 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:22:05 -0400 Subject: [PATCH 17/18] build(deps): drop numpy, ffmpeg-python and requests from commands Live listen captures the microphone as raw int16 with sounddevice alone, without numpy. debug audio now runs ffprobe itself, with the argv ffmpeg-python built (-show_format -show_streams -of json), instead of through ffmpeg-python. Both the standard probe and --ffprobe-args run from an argv list with no shell, stdin closed and a 30 s timeout. The input is opened through the file protocol only (-protocol_whitelist file), and a name that starts with "-" is passed as "./-...". The JSON output and the failure messages are unchanged. The listen, debug-audio and debug-network packages no longer declare numpy, ffmpeg-python (and with it future) or requests; the root package keeps requests until 1.0, and a plugin that relied on numpy or ffmpeg-python arriving with deepctl must now declare it itself. --- packages/deepctl-cmd-debug-audio/README.md | 1 - .../deepctl-cmd-debug-audio/pyproject.toml | 1 - .../src/deepctl_cmd_debug_audio/command.py | 65 +++- .../tests/unit/test_audio_command.py | 246 +++++++++++++-- packages/deepctl-cmd-debug-network/README.md | 1 - .../deepctl-cmd-debug-network/pyproject.toml | 1 - packages/deepctl-cmd-listen/README.md | 1 - packages/deepctl-cmd-listen/pyproject.toml | 1 - .../src/deepctl_cmd_listen/command.py | 3 +- .../tests/unit/test_listen_command.py | 198 +++++++++++- pyproject.toml | 9 +- tests/unit/dependency_allowlist.txt | 79 +++++ tests/unit/test_dependency_allowlist.py | 282 ++++++++++++++++++ tests/upgrade/lowest_direct.sh | 2 + uv.lock | 254 ---------------- 15 files changed, 830 insertions(+), 314 deletions(-) create mode 100644 tests/unit/dependency_allowlist.txt create mode 100644 tests/unit/test_dependency_allowlist.py diff --git a/packages/deepctl-cmd-debug-audio/README.md b/packages/deepctl-cmd-debug-audio/README.md index 832eb13c..ac7a1d16 100644 --- a/packages/deepctl-cmd-debug-audio/README.md +++ b/packages/deepctl-cmd-debug-audio/README.md @@ -38,7 +38,6 @@ pipx run deepctl --help - `click>=8.2.0,<9` - `rich>=13.0.0,<16` - `pydantic>=2.0.0,<3` -- `ffmpeg-python>=0.2.0` - `httpx>=0.24.0,<1` ## License diff --git a/packages/deepctl-cmd-debug-audio/pyproject.toml b/packages/deepctl-cmd-debug-audio/pyproject.toml index 57061a62..0aba4d52 100644 --- a/packages/deepctl-cmd-debug-audio/pyproject.toml +++ b/packages/deepctl-cmd-debug-audio/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "click>=8.2.0,<9", "rich>=13.0.0,<16", "pydantic>=2.0.0,<3", - "ffmpeg-python>=0.2.0", "httpx>=0.24.0,<1", ] diff --git a/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py b/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py index aec07b51..710db3d8 100644 --- a/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py +++ b/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py @@ -8,7 +8,6 @@ from typing import Any from urllib.parse import urlparse -import ffmpeg # type: ignore[import-untyped] from deepctl_core import ( AuthManager, BaseCommand, @@ -45,6 +44,32 @@ def _ffprobe_input(file_path: str) -> str: console = get_console() status_console = get_status_console() +# Seconds a single ffprobe run may take before it is abandoned. +FFPROBE_TIMEOUT_S = 30 + + +def _probe_input(path: str) -> list[str]: + """ffprobe's input arguments, always last on the command line. + + The input is always a local file (URLs are downloaded first), so ffprobe + may open it only through the file protocol: tcp:, concat:, pipe: and the + rest are refused. It is named as ``file:`` + (``_ffprobe_input``), so ffprobe reads it as a file and never as an + option, whatever it is called. + """ + return ["-protocol_whitelist", "file", _ffprobe_input(path)] + + +def _run_ffprobe(cmd: list[str]) -> "subprocess.CompletedProcess[bytes]": + """Run ffprobe from an argv list: no shell, no stdin, bounded in time.""" + return subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + capture_output=True, + timeout=FFPROBE_TIMEOUT_S, + check=False, + ) + def _human_output() -> bool: """True when stdout is for a person, not a parser.""" @@ -239,17 +264,37 @@ def run_ffprobe( # value stays one argument, and run ffprobe directly cmd = [ffprobe_bin, "-v", "quiet", "-print_format", "json"] cmd.extend(shlex.split(custom_args)) - cmd.append(_ffprobe_input(file_path)) + cmd.extend(_probe_input(file_path)) - result = subprocess.run(cmd, capture_output=True, text=True) + result = _run_ffprobe(cmd) if result.returncode != 0: - raise Exception(f"ffprobe failed: {result.stderr}") - - return dict(json.loads(result.stdout)) - else: - # Use python-ffmpeg for standard probe - probe = ffmpeg.probe(_ffprobe_input(file_path), cmd=ffprobe_bin) - return dict(probe) + stderr = result.stderr.decode("utf-8", errors="replace") + raise Exception(f"ffprobe failed: {stderr}") + + return dict(json.loads(result.stdout.decode("utf-8"))) + # The standard probe: the argv ffmpeg-python's probe() built, + # plus the file-only protocol whitelist. + cmd = [ + ffprobe_bin, + "-show_format", + "-show_streams", + "-of", + "json", + *_probe_input(file_path), + ] + probe = _run_ffprobe(cmd) + if probe.returncode != 0: + # stderr is captured, so say what ffprobe said: its last line. + said = [ + line.strip() + for line in (probe.stderr or b"") + .decode("utf-8", "replace") + .splitlines() + if line.strip() + ] + why = said[-1][:300] if said else f"exit {probe.returncode}" + raise Exception(f"ffprobe couldn't read the file: {why}") + return dict(json.loads(probe.stdout.decode("utf-8"))) except Exception as e: raise Exception(f"Failed to probe file: {e!s}") diff --git a/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py b/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py index e8680cd9..fd9ac68b 100644 --- a/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py +++ b/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py @@ -2,9 +2,15 @@ import json import os +import shutil +import socket +import subprocess +import threading +import time from unittest.mock import MagicMock, Mock, patch import pytest +from deepctl_cmd_debug_audio import command as command_module from deepctl_cmd_debug_audio.command import AudioCommand from deepctl_cmd_debug_audio.models import ( AudioDebugResult, @@ -12,7 +18,6 @@ AudioInfo, AudioStream, ) -from deepctl_cmd_debug_audio import command as command_module from deepctl_core import AuthManager, Config, DeepgramClient @@ -26,6 +31,15 @@ def _ffprobe_lookup_via_which(monkeypatch): ) +def _completed(returncode: int, stdout: str, stderr: bytes = b"") -> Mock: + """A subprocess.run result carrying ffprobe's bytes output.""" + result = Mock() + result.returncode = returncode + result.stdout = stdout.encode("utf-8") + result.stderr = stderr + return result + + class TestAudioCommand: """Test cases for AudioCommand.""" @@ -109,30 +123,103 @@ def test_check_ffmpeg_installed(self, mock_which, command): assert command.check_ffmpeg_installed() is False @patch("shutil.which", return_value="/usr/bin/ffprobe") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_run_ffprobe_standard( - self, mock_probe, mock_which, command, sample_probe_data + self, mock_run, mock_which, command, sample_probe_data ): - """Test running ffprobe without custom arguments.""" - mock_probe.return_value = sample_probe_data + """The standard probe runs ffprobe's JSON argv as a list, no shell.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.run_ffprobe("test.mp3") assert result == sample_probe_data - mock_probe.assert_called_once_with( - "file:" + os.path.abspath("test.mp3"), cmd="/usr/bin/ffprobe" + mock_run.assert_called_once() + args, kwargs = mock_run.call_args + assert args[0] == [ + "/usr/bin/ffprobe", + "-show_format", + "-show_streams", + "-of", + "json", + "-protocol_whitelist", + "file", + "file:" + os.path.abspath("test.mp3"), + ] + assert kwargs.get("shell", False) is False + assert kwargs["timeout"] == 30 + assert kwargs["capture_output"] is True + assert kwargs["stdin"] is subprocess.DEVNULL + + @pytest.mark.parametrize( + ("given", "probed"), + [ + ("-v.wav", "file:" + os.path.abspath("-v.wav")), + ("-show_entries.wav", "file:" + os.path.abspath("-show_entries.wav")), + ("/abs/-v.wav", "file:" + os.path.abspath("/abs/-v.wav")), + ("dir/-v.wav", "file:" + os.path.abspath("dir/-v.wav")), + ], + ) + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_never_passes_a_path_as_an_option( + self, mock_run, mock_which, command, sample_probe_data, given, probed + ): + """A file name that starts with "-" reaches ffprobe as a file.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) + + command.run_ffprobe(given) + assert mock_run.call_args[0][0][-3:] == ["-protocol_whitelist", "file", probed] + + command.run_ffprobe(given, "-show_streams") + assert mock_run.call_args[0][0][-3:] == ["-protocol_whitelist", "file", probed] + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_failure_says_what_ffprobe_said( + self, mock_run, mock_which, command + ): + """A non-zero ffprobe exit shows the last line ffprobe wrote.""" + mock_run.return_value = _completed( + 1, "", b"ffprobe version 7\n\nbroken.mp4: moov atom not found\n" ) + with pytest.raises(Exception) as exc: + command.run_ffprobe("broken.mp4") + + assert str(exc.value) == ( + "Failed to probe file: ffprobe couldn't read the file: " + "broken.mp4: moov atom not found" + ) + + mock_run.return_value = _completed(1, "", b"") + with pytest.raises(Exception) as exc: + command.run_ffprobe("broken.mp4") + assert str(exc.value).endswith("ffprobe couldn't read the file: exit 1") + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_timeout_is_a_probe_failure( + self, mock_run, mock_which, command + ): + """An ffprobe that hangs past the timeout is reported, not waited on.""" + import subprocess + + mock_run.side_effect = subprocess.TimeoutExpired(["ffprobe"], 30) + + with pytest.raises(Exception) as exc: + command.run_ffprobe("slow.mp4") + + assert str(exc.value).startswith("Failed to probe file: ") + assert "timed out after 30 seconds" in str(exc.value) + @patch("shutil.which", return_value="/usr/bin/ffprobe") @patch("subprocess.run") def test_run_ffprobe_custom_args( self, mock_run, mock_which, command, sample_probe_data ): - """Test running ffprobe with custom arguments.""" - mock_result = Mock() - mock_result.returncode = 0 - mock_result.stdout = json.dumps(sample_probe_data) - mock_run.return_value = mock_result + """Custom arguments go before the input, which keeps the file-only + whitelist, the timeout and the closed stdin.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.run_ffprobe("test.mp3", "-show_streams -show_format") @@ -145,23 +232,44 @@ def test_run_ffprobe_custom_args( "json", "-show_streams", "-show_format", + "-protocol_whitelist", + "file", "file:" + os.path.abspath("test.mp3"), ] mock_run.assert_called_once() assert mock_run.call_args[0][0] == expected_cmd + assert mock_run.call_args[1]["timeout"] == 30 + assert mock_run.call_args[1]["stdin"] is subprocess.DEVNULL + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_custom_args_cannot_widen_the_whitelist( + self, mock_run, mock_which, command, sample_probe_data + ): + """A user-supplied -protocol_whitelist comes before ours, so the + file-only one is the one that applies to the input.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) + + command.run_ffprobe("test.mp3", "-protocol_whitelist file,tcp") + + assert mock_run.call_args[0][0][-3:] == [ + "-protocol_whitelist", + "file", + "file:" + os.path.abspath("test.mp3"), + ] @patch("shutil.which", return_value="/usr/bin/ffprobe") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_option_like_file_name_is_probed_as_a_file( - self, mock_probe, mock_which, command, sample_probe_data, tmp_path, monkeypatch + self, mock_run, mock_which, command, sample_probe_data, tmp_path, monkeypatch ): """A file named "-f lavfi" is an input, not options.""" - mock_probe.return_value = sample_probe_data + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) monkeypatch.chdir(tmp_path) command.run_ffprobe("-f lavfi") - target = mock_probe.call_args.args[0] + target = mock_run.call_args[0][0][-1] assert target == f"file:{tmp_path}{os.sep}-f lavfi" assert not target.startswith("-") @@ -171,7 +279,7 @@ def test_custom_args_are_split_like_a_shell( self, mock_run, mock_which, command, sample_probe_data ): """--ffprobe-args keeps a quoted value as one argument (shlex).""" - mock_run.return_value = Mock(returncode=0, stdout=json.dumps(sample_probe_data)) + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) command.run_ffprobe( "-f lavfi", "-show_entries 'format=duration,size' -select_streams a" @@ -183,6 +291,8 @@ def test_custom_args_are_split_like_a_shell( "format=duration,size", "-select_streams", "a", + "-protocol_whitelist", + "file", "file:" + os.path.abspath("-f lavfi"), ] @@ -235,10 +345,10 @@ def test_handle_ffmpeg_not_installed( assert result.ffmpeg_installed is False @patch("shutil.which") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_handle_success( self, - mock_probe, + mock_run, mock_which, command, mock_config, @@ -248,7 +358,7 @@ def test_handle_success( ): """Test successful audio file analysis.""" mock_which.return_value = "/usr/bin/ffprobe" - mock_probe.return_value = sample_probe_data + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.handle( config=mock_config, @@ -264,10 +374,10 @@ def test_handle_success( assert result.audio_info.format.filename == "test.mp3" @patch("shutil.which") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_handle_probe_error( self, - mock_probe, + mock_run, mock_which, command, mock_config, @@ -276,7 +386,7 @@ def test_handle_probe_error( ): """Test handling ffprobe errors.""" mock_which.return_value = "/usr/bin/ffprobe" - mock_probe.side_effect = Exception("Failed to probe file") + mock_run.return_value = _completed(1, "", b"Invalid data found") result = command.handle( config=mock_config, @@ -382,9 +492,7 @@ def test_analysis_failure_panel_goes_to_stderr( def test_human_rendering_is_suppressed_for_machines( self, _ffmpeg, _format, command, mocks, probe_data, capsys ): - with patch.object( - AudioCommand, "run_ffprobe", return_value=probe_data - ): + with patch.object(AudioCommand, "run_ffprobe", return_value=probe_data): result = command.handle(*mocks, file="test.mp3") captured = capsys.readouterr() @@ -392,16 +500,12 @@ def test_human_rendering_is_suppressed_for_machines( assert captured.out == "" assert "Analyzing audio file" in captured.err - @patch( - "deepctl_cmd_debug_audio.command.get_output_format", return_value="default" - ) + @patch("deepctl_cmd_debug_audio.command.get_output_format", return_value="default") @patch.object(AudioCommand, "check_ffmpeg_installed", return_value=True) def test_human_rendering_still_prints_for_humans( self, _ffmpeg, _format, command, mocks, probe_data, capsys ): - with patch.object( - AudioCommand, "run_ffprobe", return_value=probe_data - ): + with patch.object(AudioCommand, "run_ffprobe", return_value=probe_data): command.handle(*mocks, file="test.mp3") captured = capsys.readouterr() @@ -423,3 +527,81 @@ def test_ffprobe_never_from_current_directory(tmp_path, monkeypatch): assert command.check_ffmpeg_installed() is False with pytest.raises(Exception, match="ffprobe not found"): command.run_ffprobe("a.wav") + + +# -- real ffprobe: the argv hardening holds against the binary itself ------- + +_FFPROBE = shutil.which("ffprobe") +needs_ffprobe = pytest.mark.skipif(_FFPROBE is None, reason="ffprobe not installed") + + +def _write_wav(path) -> None: + import wave + + with wave.open(str(path), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(16000) + w.writeframes(b"\x00\x00" * 1600) + + +@pytest.fixture +def real_command(monkeypatch): + """An AudioCommand that uses the real ffprobe on PATH.""" + monkeypatch.setattr(command_module, "resolve_tool", lambda *a: _FFPROBE) + return AudioCommand() + + +@needs_ffprobe +class TestRealFfprobe: + def test_dash_named_file_is_probed_as_a_file( + self, real_command, tmp_path, monkeypatch + ): + monkeypatch.chdir(tmp_path) + _write_wav(tmp_path / "-v.wav") + + for custom in (None, "-show_streams"): + data = real_command.run_ffprobe("-v.wav", custom) + assert data["streams"][0]["codec_name"] == "pcm_s16le" + + def test_tcp_input_is_refused_without_connecting(self, real_command): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(0.2) + port = listener.getsockname()[1] + accepted: list[object] = [] + stop = threading.Event() + + def accept() -> None: + while not stop.is_set(): + try: + accepted.append(listener.accept()[0]) + except OSError: + continue + + thread = threading.Thread(target=accept, daemon=True) + thread.start() + try: + for custom in (None, "-show_streams", "-protocol_whitelist file,tcp"): + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe(f"tcp://127.0.0.1:{port}", custom) + time.sleep(0.3) + finally: + stop.set() + thread.join(2) + listener.close() + assert accepted == [] + + def test_concat_input_is_refused(self, real_command, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write_wav(tmp_path / "a.wav") + + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe("concat:a.wav|a.wav") + + def test_pipe_input_returns_promptly(self, real_command): + start = time.monotonic() + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe("pipe:0") + assert time.monotonic() - start < 5 diff --git a/packages/deepctl-cmd-debug-network/README.md b/packages/deepctl-cmd-debug-network/README.md index 2da03160..b79e01c5 100644 --- a/packages/deepctl-cmd-debug-network/README.md +++ b/packages/deepctl-cmd-debug-network/README.md @@ -39,7 +39,6 @@ pipx run deepctl --help - `rich>=13.0.0,<16` - `pydantic>=2.0.0,<3` - `httpx>=0.24.0,<1` -- `requests>=2.31,<3.0` ## License diff --git a/packages/deepctl-cmd-debug-network/pyproject.toml b/packages/deepctl-cmd-debug-network/pyproject.toml index a37b284d..1af030f6 100644 --- a/packages/deepctl-cmd-debug-network/pyproject.toml +++ b/packages/deepctl-cmd-debug-network/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ "rich>=13.0.0,<16", "pydantic>=2.0.0,<3", "httpx>=0.24.0,<1", - "requests>=2.31,<3.0", ] [project.scripts] diff --git a/packages/deepctl-cmd-listen/README.md b/packages/deepctl-cmd-listen/README.md index 864fa2e9..e767a5a7 100644 --- a/packages/deepctl-cmd-listen/README.md +++ b/packages/deepctl-cmd-listen/README.md @@ -45,7 +45,6 @@ Status lines and errors show the URL as `scheme://host/…/`, - `websockets>=13.1,<18` - `deepgram-captions>=2.0.0` - `sounddevice>=0.4.6` -- `numpy>=1.24.0` ## License diff --git a/packages/deepctl-cmd-listen/pyproject.toml b/packages/deepctl-cmd-listen/pyproject.toml index c770b6ee..e41dc167 100644 --- a/packages/deepctl-cmd-listen/pyproject.toml +++ b/packages/deepctl-cmd-listen/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "websockets>=13.1,<18", "deepgram-captions>=2.0.0", "sounddevice>=0.4.6", - "numpy>=1.24.0", ] [project.scripts] diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index b36e567f..93c7e3ba 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -1017,7 +1017,6 @@ async def _ws_mic( ) -> BaseResult | ListenCodedError: import asyncio - import numpy as np import sounddevice as sd from . import live @@ -1062,7 +1061,7 @@ def start_mic() -> None: stream = sd.RawInputStream( samplerate=sample_rate, channels=channels, - dtype=np.int16, + dtype="int16", callback=callback, blocksize=live.frame_bytes("linear16", sample_rate, channels) // (2 * channels), diff --git a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py index f3e2da3d..10d5401a 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_listen_command.py @@ -1099,6 +1099,201 @@ def close(self): assert command.exit_code_for(result) == 0 assert '{"type": "CloseStream"}' in ws.sent + def test_ws_mic_streams_raw_int16_frames_without_numpy(self, command, monkeypatch): + """The mic path opens a raw int16 stream and forwards each callback + buffer to the socket byte for byte, with numpy unimportable.""" + import asyncio as _asyncio + import json as _json + import signal + import sys + import types + from array import array + from unittest.mock import MagicMock + + from deepctl_cmd_listen import live + + # Any `import numpy` now raises ImportError. + monkeypatch.setitem(sys.modules, "numpy", None) + + # Fixed fixture: three 4-sample blocks of native-order int16, as + # PortAudio delivers them to the callback. + samples = [0, 1, -1, 32767, -32768, 1234, -1234, 7, 8, 9, -9, 0] + blocks = [array("h", samples[i : i + 4]).tobytes() for i in range(0, 12, 4)] + + opened: dict = {} + + class _FakeRawInputStream: + def __init__(self, **kwargs): + opened.update(kwargs) + self.events: list[str] = [] + opened["stream"] = self + + def start(self): + self.events.append("start") + for block in blocks: + # sounddevice hands the callback a CFFI buffer; a + # memoryview is the same buffer-protocol object. + opened["callback"](memoryview(block), 4, None, None) + + def stop(self): + self.events.append("stop") + + def close(self): + self.events.append("close") + + fake_sd = types.ModuleType("sounddevice") + fake_sd.RawInputStream = _FakeRawInputStream + monkeypatch.setitem(sys.modules, "sounddevice", fake_sd) + + ws = _EndOnCloseStream() + _install_websockets(monkeypatch, ws) + + client = MagicMock() + client.config.get_profile.return_value.base_url = "https://api.deepgram.com" + client.auth_manager.get_api_key.return_value = "k" + + loop = _asyncio.new_event_loop() + # The user's Ctrl-C, once the three blocks are on the socket. + loop.call_later(0.2, signal.raise_signal, signal.SIGINT) + try: + loop.run_until_complete( + command._ws_mic( + client, + model="nova-3", + language="en-US", + api_version=1, + diarize=False, + smart_format=True, + punctuate=True, + interim=False, + redact=(), + numerals=False, + sample_rate=16000, + channels=1, + ) + ) + finally: + loop.close() + + audio = [frame for frame in ws.sent if isinstance(frame, bytes)] + assert opened["dtype"] == "int16" + assert opened["samplerate"] == 16000 + assert opened["channels"] == 1 + assert opened["blocksize"] == live.frame_bytes("linear16", 16000, 1) // 2 + assert audio == blocks + assert _json.loads(ws.sent[-1]) == {"type": "CloseStream"} + assert opened["stream"].events == ["start", "stop", "close"] + + def test_sigint_during_mic_capture_stops_cleanly(self, command, monkeypatch): + """A real SIGINT while frames are flowing stops the capture: the + stream is stopped and closed, CloseStream is sent, and the transcript + so far is kept on every Python version. Ctrl-C is how a mic session + ends, so the result is success, exit 0.""" + import json as _json + import os + import signal + import sys + import threading + import time + import types + from unittest.mock import MagicMock + + if signal.getsignal(signal.SIGINT) is not signal.default_int_handler: + pytest.skip("SIGINT is not Python's default handler here") + monkeypatch.setitem(sys.modules, "numpy", None) + + events: list[str] = [] + + class _PacedStream: + def __init__(self, *, callback, blocksize, samplerate, **kwargs): + self._callback = callback + self._period = blocksize / samplerate + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self._stop.wait(self._period): + self._callback(memoryview(b"\x01\x00" * 800), 800, None, None) + + def start(self): + events.append("start") + self._thread.start() + + def stop(self): + events.append("stop") + self._stop.set() + + def close(self): + events.append("close") + + fake_sd = types.ModuleType("sounddevice") + fake_sd.RawInputStream = _PacedStream + monkeypatch.setitem(sys.modules, "sounddevice", fake_sd) + + class _OneResultThenEndOnClose(_EndOnCloseStream): + def __init__(self): + super().__init__() + self._said = False + self._pressed = False + + async def send(self, data): + await super().send(data) + audio = [f for f in self.sent if isinstance(f, bytes)] + if len(audio) == 3 and not self._pressed: + # Frames are flowing: the user presses Ctrl-C, once (a + # later CloseStream send must not schedule another). + self._pressed = True + threading.Timer(0.05, os.kill, (os.getpid(), signal.SIGINT)).start() + + async def __anext__(self): + if not self._said: + self._said = True + return _json.dumps( + { + "type": "Results", + "is_final": True, + "start": 0.0, + "duration": 1.0, + "channel": { + "alternatives": [{"transcript": "hi", "words": []}] + }, + } + ) + return await super().__anext__() + + ws = _OneResultThenEndOnClose() + _install_websockets(monkeypatch, ws) + + client = MagicMock() + client.config.get_profile.return_value.base_url = "https://api.deepgram.com" + client.auth_manager.get_api_key.return_value = "k" + + start = time.monotonic() + result = command._stream_mic( + client, + model="nova-3", + language="en-US", + api_version=1, + diarize=False, + smart_format=True, + punctuate=True, + interim=False, + redact=(), + numerals=False, + sample_rate=16000, + channels=1, + save_to=None, + caption_format=None, + ) + + assert time.monotonic() - start < 5 + assert events == ["start", "stop", "close"] + assert all(isinstance(frame, bytes) for frame in ws.sent[:3]) + assert _json.loads(ws.sent[-1]) == {"type": "CloseStream"} + assert result.status == "success" + assert result.transcript == "hi" + assert command.exit_code_for(result) == 0 + def test_end_of_turn_with_empty_transcript_is_noop(self, command, capsys): """An EndOfTurn that never carried text emits nothing (no blank line).""" acc: list[str] = [] @@ -1214,10 +1409,7 @@ def close(self): fake_sounddevice = types.ModuleType("sounddevice") fake_sounddevice.RawInputStream = _FakeInputStream - fake_numpy = types.ModuleType("numpy") - fake_numpy.int16 = "int16" monkeypatch.setitem(sys.modules, "sounddevice", fake_sounddevice) - monkeypatch.setitem(sys.modules, "numpy", fake_numpy) client = MagicMock() client.config.get_profile.return_value.base_url = "https://api.deepgram.com" diff --git a/pyproject.toml b/pyproject.toml index 08d77a05..41774642 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,15 +155,10 @@ disable_error_code = ["call-arg", "no-untyped-call", "no-untyped-def", "untyped- module = [ "sounddevice", "sounddevice.*", - "numpy", - "numpy.*", ] -# Optional microphone dependencies (sounddevice, numpy) ship without type -# stubs. +# The microphone dependency (sounddevice) ships without type stubs. Skip +# analyzing it; usages fall back to Any. ignore_missing_imports = true -# numpy>=2.2 ships PEP 695 `type` aliases in its stubs, which mypy rejects -# under python_version = "3.10" (aborting before our code is checked). Skip -# analyzing these third-party stubs entirely; usages fall back to Any. follow_imports = "skip" follow_imports_for_stubs = true diff --git a/tests/unit/dependency_allowlist.txt b/tests/unit/dependency_allowlist.txt new file mode 100644 index 00000000..ea295a41 --- /dev/null +++ b/tests/unit/dependency_allowlist.txt @@ -0,0 +1,79 @@ +# Third-party distributions the 30 lockstep packages may pull in at runtime, +# on any platform or Python version uv.lock covers (PEP 503 names, sorted). +# Checked by tests/unit/test_dependency_allowlist.py. Adding a runtime +# dependency, direct or transitive, means adding it here in the same PR. +# numpy, ffmpeg-python and future were dropped in 0.5.0 and are refused +# even if listed. +# Format: one name per line, optionally followed by flags. The only flag is +# `floors-only`: the name is reached only when a dependency sits at its +# declared floor (the lowest-direct CI job), not through uv.lock, and that +# job checks it is really installed there. Text after "#" is a comment. +aiohappyeyeballs +aiohttp +aiosignal +annotated-types +anyio +async-timeout +attrs +backports-tarfile +certifi +cffi +charset-normalizer +click +colorama +cryptography +deepgram-captions +deepgram-sdk +exceptiongroup +frozenlist +h11 +httpcore +httpx +httpx-sse +idna +importlib-metadata +jaraco-classes +jaraco-context +jaraco-functools +jeepney +jsonschema +jsonschema-specifications +keyring +markdown-it-py +mcp +mdurl +more-itertools +multidict +packaging +platformdirs +propcache +pycparser +pydantic +pydantic-core +pydantic-settings +pygments +pyjwt +pyparsing floors-only # via packaging 21.x +python-dotenv +python-multipart +pywin32 +pywin32-ctypes +pyyaml +referencing +requests +rich +rpds-py +secretstorage +sentry-sdk +sniffio floors-only # via older anyio/httpx +sounddevice +sse-starlette +starlette +toml +typing-extensions +typing-inspection +urllib3 +uvicorn +websockets +yarl +zipp diff --git a/tests/unit/test_dependency_allowlist.py b/tests/unit/test_dependency_allowlist.py new file mode 100644 index 00000000..c2e24353 --- /dev/null +++ b/tests/unit/test_dependency_allowlist.py @@ -0,0 +1,282 @@ +"""The third-party closure of the 30 lockstep packages is allowlisted. + +Every distribution a plain `pip install deepctl` can pull in, on any +platform or Python the lock covers, must be named in +`dependency_allowlist.txt`. A new runtime dependency, direct or transitive, +fails here until someone adds it on purpose. numpy, ffmpeg-python and future +were dropped in 0.5.0 (the install diet) and must not come back, allowlisted +or not. + +Two views of the closure are checked: + +* the lock: `uv.lock`'s runtime `dependencies` for every lockstep package, + across all markers, followed transitively (extras included). This is the + same graph the Requires-Dist metadata describes, read without installing. +* the installed environment: Requires-Dist of the installed lockstep + distributions, evaluated for this interpreter and followed transitively. +""" + +from __future__ import annotations + +import os +import re +import sys +from importlib import metadata +from pathlib import Path +from typing import Any + +import pytest + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - only taken on Python 3.10 + import tomli as tomllib + +REPO = Path(__file__).resolve().parents[2] +ALLOWLIST = Path(__file__).with_name("dependency_allowlist.txt") +ROOT_NAME = "deepctl" +# Versioned on its own, the way an outside plugin would be. +NOT_LOCKSTEP = {"deepctl-plugin-example"} +FORBIDDEN = {"numpy", "ffmpeg-python", "future"} + + +def normalize(name: str) -> str: + """PEP 503 normalized distribution name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def lockstep_names(repo: Path = REPO) -> set[str]: + """The root and every first-party sibling that versions with it.""" + names = {ROOT_NAME} + for pyproject in sorted((repo / "packages").glob("*/pyproject.toml")): + with pyproject.open("rb") as fh: + name = normalize(tomllib.load(fh)["project"]["name"]) + if name not in NOT_LOCKSTEP: + names.add(name) + return names + + +FLAGS = {"floors-only"} +#: Set by tests/upgrade/lowest_direct.sh: this process runs in the +#: environment resolved at the declared floors. +LOWEST_DIRECT_ENV = "DEEPCTL_LOWEST_DIRECT" + + +def parse_allowlist(path: Path = ALLOWLIST) -> list[tuple[str, frozenset[str]]]: + """(name, flags) per entry, in file order, without comments or blanks.""" + entries = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + fields = line.split("#", 1)[0].split() + if not fields: + continue + name, flags = fields[0], frozenset(fields[1:]) + unknown = flags - FLAGS + if unknown: + raise ValueError(f"{path.name}:{number}: unknown flag(s) {sorted(unknown)}") + entries.append((name, flags)) + return entries + + +def read_allowlist(path: Path = ALLOWLIST) -> list[str]: + """Allowlisted names, in file order.""" + return [name for name, _ in parse_allowlist(path)] + + +def floors_only(path: Path = ALLOWLIST) -> set[str]: + """Entries reached only at declared floors, not through uv.lock.""" + return {name for name, flags in parse_allowlist(path) if "floors-only" in flags} + + +def lock_closure(lock: dict[str, Any], roots: set[str]) -> set[str]: + """Third-party names reachable from `roots` through runtime deps.""" + by_name: dict[str, list[dict[str, Any]]] = {} + for package in lock["package"]: + by_name.setdefault(normalize(package["name"]), []).append(package) + + seen: set[tuple[str, str | None]] = set() + todo: list[tuple[str, str | None]] = [(root, None) for root in roots] + reached: set[str] = set() + while todo: + name, extra = todo.pop() + if (name, extra) in seen: + continue + seen.add((name, extra)) + reached.add(name) + for package in by_name.get(name, []): + if extra is None: + deps = package.get("dependencies", []) + else: + deps = package.get("optional-dependencies", {}).get(extra, []) + for dep in deps: + dep_name = normalize(dep["name"]) + todo.append((dep_name, None)) + todo.extend((dep_name, e) for e in dep.get("extra", [])) + return reached - roots + + +def installed_closure(roots: set[str]) -> set[str]: + """Third-party names reachable from the installed `roots` through + Requires-Dist, with markers evaluated for this interpreter.""" + from packaging.requirements import Requirement + + seen: set[tuple[str, str]] = set() + todo: list[tuple[str, str]] = [(root, "") for root in roots] + reached: set[str] = set() + while todo: + name, extra = todo.pop() + if (name, extra) in seen: + continue + seen.add((name, extra)) + try: + requires = metadata.requires(name) or [] + except metadata.PackageNotFoundError: + # A marker can name a dist this platform never installs. + continue + reached.add(name) + for raw in requires: + req = Requirement(raw) + if req.marker is not None and not req.marker.evaluate({"extra": extra}): + continue + dep = normalize(req.name) + todo.append((dep, "")) + todo.extend((dep, e) for e in req.extras) + return reached - roots + + +def violations(closure: set[str], allowlist: set[str]) -> list[str]: + """Why the closure fails: unlisted names, then forbidden ones.""" + problems = [ + f"{name} is a runtime dependency but is not in {ALLOWLIST.name}" + for name in sorted(closure - allowlist) + ] + problems += [ + f"{name} was dropped in 0.5.0 and must not come back" + for name in sorted(closure & FORBIDDEN) + ] + return problems + + +@pytest.fixture(scope="module") +def lock() -> dict[str, Any]: + with (REPO / "uv.lock").open("rb") as fh: + return tomllib.load(fh) + + +@pytest.fixture(scope="module") +def roots() -> set[str]: + names = lockstep_names() + assert len(names) == 30, sorted(names) + return names + + +def test_allowlist_is_sorted_unique_and_normalized() -> None: + entries = read_allowlist() + assert entries == sorted(set(entries)), "keep the allowlist sorted, no repeats" + assert all(entry == normalize(entry) for entry in entries) + assert not FORBIDDEN & set(entries) + + +def test_locked_closure_is_allowlisted(lock: dict[str, Any], roots: set[str]) -> None: + closure = lock_closure(lock, roots) + assert violations(closure, set(read_allowlist())) == [] + + +def test_allowlist_has_no_stale_entries(lock: dict[str, Any], roots: set[str]) -> None: + """An entry nothing depends on any more is removed, so the list stays a + true picture of what installs.""" + stale = set(read_allowlist()) - lock_closure(lock, roots) - floors_only() + assert not stale, f"no longer a runtime dependency: {sorted(stale)}" + + +def test_installed_closure_is_allowlisted(roots: set[str]) -> None: + missing = sorted(r for r in roots if not _installed(r)) + if missing: + pytest.skip(f"lockstep packages not installed here: {missing}") + closure = installed_closure(roots) + assert violations(closure, set(read_allowlist())) == [] + + +def test_floors_only_entries_are_installed_at_the_floors(roots: set[str]) -> None: + """In the lowest-direct environment, every floors-only entry must really + be pulled in; one that is not is stale and comes off the list.""" + if os.environ.get(LOWEST_DIRECT_ENV) != "1": + pytest.skip(f"only in the lowest-direct job ({LOWEST_DIRECT_ENV}=1)") + missing = sorted(floors_only() - installed_closure(roots)) + assert not missing, f"floors-only but not reached at the floors: {missing}" + + +def test_unknown_flags_are_refused(tmp_path: Path) -> None: + listing = tmp_path / "allow.txt" + listing.write_text("idna\nsix floors-onyl # typo\n", encoding="utf-8") + with pytest.raises(ValueError, match="unknown flag"): + parse_allowlist(listing) + + +def test_flags_and_comments_parse(tmp_path: Path) -> None: + listing = tmp_path / "allow.txt" + listing.write_text("# header\nidna\nsix floors-only # why\n\n", encoding="utf-8") + assert parse_allowlist(listing) == [ + ("idna", frozenset()), + ("six", frozenset({"floors-only"})), + ] + + +def _installed(name: str) -> bool: + try: + metadata.distribution(name) + except metadata.PackageNotFoundError: + return False + return True + + +# -- negative fixtures: the check catches what it is for ------------------- + + +def _planted(lock: dict[str, Any], extra_dep: str) -> dict[str, Any]: + """A copy of the lock where deepctl-core gains `extra_dep`, which in turn + depends on six (a transitive addition).""" + packages = [] + for package in lock["package"]: + package = dict(package) + if package["name"] == "deepctl-core": + package["dependencies"] = [ + *package.get("dependencies", []), + {"name": extra_dep}, + ] + packages.append(package) + packages.append( + {"name": extra_dep, "version": "1.0", "dependencies": [{"name": "six"}]} + ) + packages.append({"name": "six", "version": "1.17.0"}) + return {**lock, "package": packages} + + +def test_a_planted_dependency_fails(lock: dict[str, Any], roots: set[str]) -> None: + allowlist = set(read_allowlist()) + closure = lock_closure(_planted(lock, "left-pad-py"), roots) + problems = violations(closure, allowlist) + assert any(p.startswith("left-pad-py ") for p in problems) + if "six" not in allowlist: + assert any(p.startswith("six ") for p in problems) + + +@pytest.mark.parametrize("dropped", sorted(FORBIDDEN)) +def test_a_dropped_dependency_fails_even_if_allowlisted( + lock: dict[str, Any], roots: set[str], dropped: str +) -> None: + allowlist = set(read_allowlist()) | {dropped} + closure = lock_closure(_planted(lock, dropped), roots) + assert violations(closure, allowlist | {"six"}) == [ + f"{dropped} was dropped in 0.5.0 and must not come back" + ] + + +def test_extras_are_followed(lock: dict[str, Any], roots: set[str]) -> None: + """A dependency requested with an extra pulls in that extra's deps.""" + planted = _planted(lock, "pkg-with-extra") + for package in planted["package"]: + if package["name"] == "deepctl-core": + package["dependencies"][-1] = {"name": "pkg-with-extra", "extra": ["x"]} + if package["name"] == "pkg-with-extra": + package["optional-dependencies"] = {"x": [{"name": "only-via-extra"}]} + assert "only-via-extra" in lock_closure(planted, roots) diff --git a/tests/upgrade/lowest_direct.sh b/tests/upgrade/lowest_direct.sh index 3b4ddd9f..a5b0edb3 100755 --- a/tests/upgrade/lowest_direct.sh +++ b/tests/upgrade/lowest_direct.sh @@ -20,6 +20,8 @@ DIST=$(cd "$1" && pwd); PY=${2:-python3}; PHASE=${3:-all} HERE=$(cd "$(dirname "$0")" && pwd) ENV=${LOWEST_ENV:-${TMPDIR:-/tmp}/lowest-direct} export DEEPCTL_TELEMETRY_DISABLED=1 DEEPCTL_TELEMETRY_DSN=http://k@127.0.0.1:9/1 +# tests/unit/test_dependency_allowlist.py checks its floors-only entries here. +export DEEPCTL_LOWEST_DIRECT=1 TEST_TOOLS="pytest pytest-asyncio pytest-mock pytest-timeout responses" if [ "$PHASE" != test ]; then diff --git a/uv.lock b/uv.lock index 83cbfddb..4b9e3c2e 100644 --- a/uv.lock +++ b/uv.lock @@ -1168,7 +1168,6 @@ dependencies = [ { name = "click" }, { name = "deepctl" }, { name = "deepctl-core" }, - { name = "ffmpeg-python" }, { name = "httpx" }, { name = "pydantic" }, { name = "rich" }, @@ -1179,7 +1178,6 @@ requires-dist = [ { name = "click", specifier = ">=8.2.0,<9" }, { name = "deepctl", editable = "." }, { name = "deepctl-core", editable = "packages/deepctl-core" }, - { name = "ffmpeg-python", specifier = ">=0.2.0" }, { name = "httpx", specifier = ">=0.24.0,<1" }, { name = "pydantic", specifier = ">=2.0.0,<3" }, { name = "rich", specifier = ">=13.0.0,<16" }, @@ -1229,7 +1227,6 @@ dependencies = [ { name = "deepctl-core" }, { name = "httpx" }, { name = "pydantic" }, - { name = "requests" }, { name = "rich" }, ] @@ -1240,7 +1237,6 @@ requires-dist = [ { name = "deepctl-core", editable = "packages/deepctl-core" }, { name = "httpx", specifier = ">=0.24.0,<1" }, { name = "pydantic", specifier = ">=2.0.0,<3" }, - { name = "requests", specifier = ">=2.31,<3.0" }, { name = "rich", specifier = ">=13.0.0,<16" }, ] @@ -1384,9 +1380,6 @@ dependencies = [ { name = "deepctl-core" }, { name = "deepctl-shared-utils" }, { name = "deepgram-captions" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pydantic" }, { name = "rich" }, { name = "sounddevice" }, @@ -1401,7 +1394,6 @@ requires-dist = [ { name = "deepctl-core", editable = "packages/deepctl-core" }, { name = "deepctl-shared-utils", editable = "packages/deepctl-shared-utils" }, { name = "deepgram-captions", specifier = ">=2.0.0" }, - { name = "numpy", specifier = ">=1.24.0" }, { name = "pydantic", specifier = ">=2.0.0,<3" }, { name = "rich", specifier = ">=13.0.0,<16" }, { name = "sounddevice", specifier = ">=0.4.6" }, @@ -1883,18 +1875,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] -[[package]] -name = "ffmpeg-python" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "future" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/5e/d5f9105d59c1325759d838af4e973695081fbbc97182baf73afc78dec266/ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127", size = 21543, upload-time = "2019-07-06T00:19:08.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/0c/56be52741f75bad4dc6555991fabd2e07b432d333da82c11ad701123888a/ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5", size = 25024, upload-time = "2019-07-06T00:19:07.215Z" }, -] - [[package]] name = "filelock" version = "3.32.3" @@ -2025,15 +2005,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] -[[package]] -name = "future" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -2707,231 +2678,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] -[[package]] -name = "numpy" -version = "2.2.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, - { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, - { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, - { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, - { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, - { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, - { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, - { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, - { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, - { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, - { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, - { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, - { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, - { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, - { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, - { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, - { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, - { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, - { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, - { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, - { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, - { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, - { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, - { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, - { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, - { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, - { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, - { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, - { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, - { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, - { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, - { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, - { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, - { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, - { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, - { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, - { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, - { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, - { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, - { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, - { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, - { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, - { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, - { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, - { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, - { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, - { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, - { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, -] - -[[package]] -name = "numpy" -version = "2.4.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*'", -] -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, - { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, - { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, - { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, - { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, - { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, - { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, - { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, - { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, - { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, - { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, - { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, - { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, - { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, - { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, - { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, - { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, - { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, - { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, - { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, - { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, - { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, - { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, - { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, - { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, - { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, - { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, - { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, - { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, - { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, - { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, - { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, -] - -[[package]] -name = "numpy" -version = "2.5.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version == '3.14.*'", - "python_full_version >= '3.12' and python_full_version < '3.14'", -] -sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/72/dccb0aaf40972777283303919f613964227266d0c13adebb79ac124f1c3e/numpy-2.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14e373cfc6387177e8409dac3c7159be8eb05cd77096cd7c950268b86f62831c", size = 16891693, upload-time = "2026-08-09T13:44:51.702Z" }, - { url = "https://files.pythonhosted.org/packages/60/2e/b5aee50a1f74ac815cf8331812cb8251e29024025de462e0c047641c614c/numpy-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbd96c833ecc8cc069ce518078fc8c60cb9cbfb0fea5b7a803ad65035596d03", size = 11903109, upload-time = "2026-08-09T13:44:55.501Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f4/29e78102a80601cf034d4e9767022cffeca2c3b4c926e1754572ca95593d/numpy-2.5.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:6e8172ddfcf5cf74b811d372b570b83c60bd2de87a6fbfbebdadb4a9bd9c6cbb", size = 5350202, upload-time = "2026-08-09T13:44:58.401Z" }, - { url = "https://files.pythonhosted.org/packages/11/4b/dcd3b7eadaf4035d2c7a4289d232523a6964f602598ef7674e4bd7291f93/numpy-2.5.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f188481f1669e26f62b701e8205d19e460fa4a9b52a1414ba382330e4a3414", size = 6687736, upload-time = "2026-08-09T13:45:00.813Z" }, - { url = "https://files.pythonhosted.org/packages/e5/21/4947e0e9d6c9fc2e2ff15b8949049ee44f63adb9cacc729ab8793f97e712/numpy-2.5.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee9c4eeb8454b3660a8b53493563c3e121c2fc94fbd72b848ef814ed7b676a9", size = 15612696, upload-time = "2026-08-09T13:45:04.151Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5f/62d28cf019460c7f1394105b4d49d9911a9c444cb77ab0bd95a204c5a6de/numpy-2.5.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdec01fa790a186d430433fdd4d4ffb70eed6f0eeb4bf05c8dbe2dce0a9bcb8", size = 16722264, upload-time = "2026-08-09T13:45:07.714Z" }, - { url = "https://files.pythonhosted.org/packages/14/25/3f0be4c1b9fdf5dd5e708a6806978564d7c46a055c000496309ff2a2f8af/numpy-2.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7999d4ddb0c4025018373fd787510d46e04c769467af22869707b3c1cfd459ab", size = 16974396, upload-time = "2026-08-09T13:45:11.316Z" }, - { url = "https://files.pythonhosted.org/packages/22/72/6262cbdeeb45da9d971e40715f579d791603ba8ec0b5e2db1ac55454421d/numpy-2.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1f017dc0875c9209d219f97feceb7d54c2661bb243deb4114478e1295808af7", size = 18476044, upload-time = "2026-08-09T13:45:14.869Z" }, - { url = "https://files.pythonhosted.org/packages/36/33/29208b8b075bde62d26a81d14b358c42b0f69b6cabd98d4ff97f37f22b05/numpy-2.5.2-cp312-cp312-win32.whl", hash = "sha256:d6a48072864e3324e194a8fbb3c657bcc5b5c869dbc64c9537b1d5c862572c0a", size = 6072817, upload-time = "2026-08-09T13:45:17.867Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b9/87fea2769fe1c47c1b5b01d8310772c9d1a85d485de7cf386ef7a3332b02/numpy-2.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:28ac63476ec7651484215ee7fa15a1f78b57c14621f01e392afe17b9a1390ce4", size = 12464674, upload-time = "2026-08-09T13:45:20.734Z" }, - { url = "https://files.pythonhosted.org/packages/14/52/032b97e00461ab0809bbe4c588b035620e5a14b8cdee47ecddefc7b17d33/numpy-2.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:27650bb0e7140fa3d37b9923b4803645e0b125d190f326eecfd3f4dad8e8ade1", size = 10397131, upload-time = "2026-08-09T13:45:23.73Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d2/6b24738a0ef4557d189b150046cd07823c50e4273e8aebd651222e24306f/numpy-2.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8e4cb9a754c8a0c62eaa88273a5fba3391f4a610d1dee893c0755da31c083f15", size = 16886595, upload-time = "2026-08-09T13:45:27.323Z" }, - { url = "https://files.pythonhosted.org/packages/65/60/f2d208d366f263f39c6e69ed309290717aab41078b6d04c9be2a84fa2a07/numpy-2.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:52c808f96484f5571a5cc863775ce50247c17dfb3b0361f8ed6b4b0456f80080", size = 11896845, upload-time = "2026-08-09T13:45:31.638Z" }, - { url = "https://files.pythonhosted.org/packages/3c/79/81e0bf24f4d020a2b1d5cd297a9f60c3f24eeb116f9bba5870443f7b6a4a/numpy-2.5.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:29d81e97f668489cba8ebfd796b9bdd453525d35dd9e162e2daec94bf3fc7740", size = 5343880, upload-time = "2026-08-09T13:45:34.373Z" }, - { url = "https://files.pythonhosted.org/packages/ba/cc/e3141cf06d1a8a2c7e107543fe1269c1d1af760d4d683c0794a4ee1127c2/numpy-2.5.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afb3f0632d6b2e3ba04dbce8d1e48d321b369138b73830b5ca371a0e8d479d56", size = 6682264, upload-time = "2026-08-09T13:45:36.7Z" }, - { url = "https://files.pythonhosted.org/packages/29/f1/2a64a307d92c5d98f5255a4014eb43bb6103ee477087b61ecae44a3aa9b9/numpy-2.5.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0aadf13b60048d501e05fa699efaf7734e2494f3498a4c2a5521d822640324f3", size = 15609566, upload-time = "2026-08-09T13:45:39.518Z" }, - { url = "https://files.pythonhosted.org/packages/7b/44/59a1eb68e773c4098d107ef34a0dbdeca501d72ffcfbff9a7707343921ce/numpy-2.5.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29b86ff8a6cc556b47ec6b64b194815cc80e6bf5eedcc6cddfd65318cb0b4eee", size = 16709995, upload-time = "2026-08-09T13:45:43.661Z" }, - { url = "https://files.pythonhosted.org/packages/8a/4c/3e54d4ddbc359a1295f8b633e8106bcd4d7d4a206e82df051bdfb3058755/numpy-2.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6950c4b7dd562453090548ba7f5da7e59f57f85663f15d5dcc60e249192f7e59", size = 16972511, upload-time = "2026-08-09T13:45:47.094Z" }, - { url = "https://files.pythonhosted.org/packages/f2/9f/02e371638ebf19b66d46231e4be52999e87f32d1961b113bc45656608b22/numpy-2.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9727f472d2f3888053b8a75ab0cb94745a9de224bb5846dbadc0092101bc71d", size = 18465609, upload-time = "2026-08-09T13:45:50.808Z" }, - { url = "https://files.pythonhosted.org/packages/eb/ae/ad6645abc7a3510fe48e8ea1ab4598166f500057ef4ebf38bfad4f1577de/numpy-2.5.2-cp313-cp313-win32.whl", hash = "sha256:4f9744f9fbdcea0bc552e8f19e1f141f811a3f9bc2be2cc6e86d982cab23e3f4", size = 6070204, upload-time = "2026-08-09T13:45:54.111Z" }, - { url = "https://files.pythonhosted.org/packages/15/20/f3489f86d81ea460b2bcdceaed094142ca6579f6be0ec527b781d39afe68/numpy-2.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:85aaccb24182c25df891ad0ec333585967e115269d5f1b17f2c9ae005bc96657", size = 12460532, upload-time = "2026-08-09T13:45:57.167Z" }, - { url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f8/c3b222bf075b50afd8e949a07a15c4b312a4a84bd8102a332bcd953cbbb4/numpy-2.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d787cf769c3baeb5f6235e778edb52c08dfa923789b5958f28e6450f96107cb1", size = 16885180, upload-time = "2026-08-09T13:46:03.939Z" }, - { url = "https://files.pythonhosted.org/packages/17/e1/2c1d4b1987795a92b5bbf7c24fe249ab96aa2573ab0d7604802c189d7b86/numpy-2.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:24b9dc2e3d84aa58523798805194e23e736f3f6ce2d1a5b92583ae734e6dbda8", size = 11907878, upload-time = "2026-08-09T13:46:07.045Z" }, - { url = "https://files.pythonhosted.org/packages/b9/ee/d08226fc858044355983a6e5b94f08ff6f3969e0a2b160a4a89f0ddb3445/numpy-2.5.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9e9413326d726c2545bfa65d2c0876871e8d8386e77f992c1d426e180bbd4323", size = 5354922, upload-time = "2026-08-09T13:46:10.04Z" }, - { url = "https://files.pythonhosted.org/packages/94/f0/6d3d933056440ebbc5e6bad92065fc6c26a48a84a36b1208580e94eea76c/numpy-2.5.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:60e902ac295855348a5ca2ea4c89108989a9f5fddfad3dfc0a8f36b10358567e", size = 6679168, upload-time = "2026-08-09T13:46:12.275Z" }, - { url = "https://files.pythonhosted.org/packages/c4/3b/ecd49dd90033cceb2704d88ca905d4d7d89b0e8c739608754ffd325fa820/numpy-2.5.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e500dc868e9313530ce12ba470fe50ff3afe3d62993ed6eff652dacd555b65", size = 15624501, upload-time = "2026-08-09T13:46:15.322Z" }, - { url = "https://files.pythonhosted.org/packages/c7/99/461bd36dbdfac6c1c53efa370bd55a83227542d0d118f1677dbf1a3dacd5/numpy-2.5.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318b9a4c845dbea06708a29c84ee429cc3065048db34cdb799047643492050ee", size = 16713701, upload-time = "2026-08-09T13:46:18.949Z" }, - { url = "https://files.pythonhosted.org/packages/f9/9c/2b251df9e8a5d647b62b0cbc1b90a91850c1cf4859ecb532fd0b4eacff6c/numpy-2.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34c319e2963be042673fb46570501b2f06c41924e17e3563d58646b4380dfb68", size = 16986065, upload-time = "2026-08-09T13:46:23.006Z" }, - { url = "https://files.pythonhosted.org/packages/8f/25/20de43f53ff1390534a124475055a19f01fe10c920a0fd11b8e18d6d6052/numpy-2.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f06571a052127dc1b4e8b83029b4d1b20daa2b64a31cdd181fc6bc774e9000eb", size = 18470031, upload-time = "2026-08-09T13:46:27.102Z" }, - { url = "https://files.pythonhosted.org/packages/56/5e/0c577ca308d6da5eb79b546ba10bbe5b60148192194e2da060913b1de4f1/numpy-2.5.2-cp314-cp314-win32.whl", hash = "sha256:2cc779226e476d1e1f08c74068c419e60f41a9e0e069c92f6671d31d5c985e98", size = 6121028, upload-time = "2026-08-09T13:46:30.046Z" }, - { url = "https://files.pythonhosted.org/packages/15/5c/7bcbd5b11f94199073320410cddcbb80cee62415bfeb540874b265c2d922/numpy-2.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:7587f53dfbd5edc0f7b87c6217b4c6d2d1f2ef9c3da70bc1315e7db5f8d7ec9d", size = 12597627, upload-time = "2026-08-09T13:46:32.886Z" }, - { url = "https://files.pythonhosted.org/packages/87/bc/4d0b06fba0da90ccc75af62823cb9dcedb6c9ea0cffa058cb2c9ee773a77/numpy-2.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:3e4c367352d3747784248a227fbec218e193b56f7e6692e3b64fc805478ecfdf", size = 10680414, upload-time = "2026-08-09T13:46:36.036Z" }, - { url = "https://files.pythonhosted.org/packages/cd/17/f429aac9dc08833a0d0f188eba38c532a751b1a1f2ca6018a37b455cb321/numpy-2.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b879fb674276e331513fb136b78dbc6bd3c848309e0d841cfd63be3896c4cfc1", size = 12026967, upload-time = "2026-08-09T13:46:39.084Z" }, - { url = "https://files.pythonhosted.org/packages/ca/9f/d0849de96a2a4ceaa16662f18ee13eaa9c0aa418269fdc8c4857c56b11da/numpy-2.5.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:fd0d703772bba096843785bd38371e31bb4a0c1151497ad5739d182114a73f7f", size = 5473874, upload-time = "2026-08-09T13:46:42.075Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/8df216d4a4a5422a3de045301cf7df8ea47286d76f5cb7160b0128ac26b7/numpy-2.5.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:3a2f061cebd9e3d23bdcfaaded5e2293a4c6a5b60fa42df85d410a725ce621bf", size = 6789276, upload-time = "2026-08-09T13:46:44.387Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3a/20d7e9891c4ddfadd6ff8d95bf4b29f353d8e1770553de2099880551dfb9/numpy-2.5.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6df895598c0edcb41030126c89e0f353b07d93238116143b7405e937359736c4", size = 15659154, upload-time = "2026-08-09T13:46:47.538Z" }, - { url = "https://files.pythonhosted.org/packages/aa/d6/f3aa3d2688bf501b858835c6bd087ae9b51a56ae6fca8e2b0990abd177af/numpy-2.5.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ab3d4a901f844ea836c3e80bf463c6a27d7f3c14e8e292fcf28d348b25b9bce", size = 16748909, upload-time = "2026-08-09T13:46:51.442Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8f/1c5cae8d2baf86ab802ae97a00be55bc7e21ebc11b12bbc33376c5f05342/numpy-2.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cebc2d6dbb605a7703d59751dea4bd6b0ab127a5a4338a6f432df1936fef8b26", size = 17027685, upload-time = "2026-08-09T13:46:55.095Z" }, - { url = "https://files.pythonhosted.org/packages/5c/27/71d3467404aedc1c24ce79610f91b52b0b0f466c43a701aa56fc75c145ab/numpy-2.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eaca7ff36f0f52e2111ec71f169d8fd3e889e7ddc0d2592e0d703fd8d3ce8fac", size = 18501181, upload-time = "2026-08-09T13:46:59.09Z" }, - { url = "https://files.pythonhosted.org/packages/14/2f/42921d27c40aea7e077f4a423ae509fd9220b028cd787bafefd8ab2b3a5f/numpy-2.5.2-cp314-cp314t-win32.whl", hash = "sha256:ddf47472af2e4280d79bac82304f5e80150211f1b9e614b760061d5fdfbb6eba", size = 6271085, upload-time = "2026-08-09T13:47:01.903Z" }, - { url = "https://files.pythonhosted.org/packages/75/e6/bad5f5d56de9b1971bac959963dda276d35c40f1854475005434bbe08692/numpy-2.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:44ef9675d908e65f9953063837c3277730f3f4437615a4cdab67b366cabaf884", size = 12787971, upload-time = "2026-08-09T13:47:04.963Z" }, - { url = "https://files.pythonhosted.org/packages/df/05/f608795cb34391acd67e38d94a3c36abd8d8576293a3a80727d7595c372c/numpy-2.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:eaa088384c46f519dacb93b7ec483a6d6b19a4a2085ae4f25ab9b1c43d387d1e", size = 10750306, upload-time = "2026-08-09T13:47:07.976Z" }, - { url = "https://files.pythonhosted.org/packages/33/c6/28de0191c5f82b7d42a0a51390ba98587048aa93a39fafb05bdbe6e8d00c/numpy-2.5.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:078f9b027b478c9379b9677babbf0f8b8f1ecfada27636d7b9a93990c638739f", size = 16885274, upload-time = "2026-08-09T13:47:11.439Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d1/973ca116000d244897e468ea1aff30b589e5022e3c8744b71706fe33bd57/numpy-2.5.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:50a68f4bacd8a2b33d8da3d2269d0d78500f86ea582e4786dc10f5ef2c2c6842", size = 11907846, upload-time = "2026-08-09T13:47:15.128Z" }, - { url = "https://files.pythonhosted.org/packages/78/d9/8c4b3937ef204cb2fd88d389ccd0f265a2ffb11f35a01d2064cf46714bd6/numpy-2.5.2-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:e79aba74ffaf5f78a050d777c184cddf8fdffabab38acf5f3ef1fecbc17895d6", size = 5354892, upload-time = "2026-08-09T13:47:18.07Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/b6ee65ea2999fdb7023935e108e6fb776ee4082aa15f159acfa857e578c8/numpy-2.5.2-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:9a0731745a72a184490a582fb4af2533512bd071ace67785b5fdffc0ae58dce8", size = 6679309, upload-time = "2026-08-09T13:47:20.456Z" }, - { url = "https://files.pythonhosted.org/packages/43/f3/acb18d8b137a393c8e7803a8c994c9e64bde3930692a69d826993113a159/numpy-2.5.2-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ec954036759bcee3aa484f8603bd9c14f3e776293b85578b8734c2d72777c69", size = 15625850, upload-time = "2026-08-09T13:47:24.365Z" }, - { url = "https://files.pythonhosted.org/packages/a9/bf/a8e9bb0db815a0e265b5744ebedd3af0bd5faad8604e5b50a1cd012f3c91/numpy-2.5.2-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc649493697006bc90614a5f0bbc8cb3cb1866715c474e473694968d7e6b99ab", size = 16713664, upload-time = "2026-08-09T13:47:27.965Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c3/6e913736b3dd6582344af32418b5fb9dab34282e8a8174ae1d54ceb0fc13/numpy-2.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cf7de32f486e4ac9e2d93b810f9e9ac72a728dd46a32a0bb403222f27f653514", size = 16986749, upload-time = "2026-08-09T13:47:31.541Z" }, - { url = "https://files.pythonhosted.org/packages/80/09/7d3b23eff5c7428ef6c01e6f7052bb60d504c4d33e317b36b8959c24ad97/numpy-2.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2ffa7bacab3e2ee1b19ed31766bb60bb380b68c23f051e199c5cc598afd68710", size = 18470495, upload-time = "2026-08-09T13:47:35.364Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a4/68a321d825374f6eb677ffe8ef8c6b9a328304e6fd2e39d9530822776607/numpy-2.5.2-cp315-cp315-win32.whl", hash = "sha256:6b588cc8f902d6bff201c19fd00c43ab8545671e3554d014e12e14139e5e8617", size = 6120696, upload-time = "2026-08-09T13:47:38.561Z" }, - { url = "https://files.pythonhosted.org/packages/c8/23/deafbb1700f79fae9cd1e91220f133d124cc267de1b584da3fbf6db2f6cd/numpy-2.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:07d4e89f3a9ab0a9ba24264ccdb642b3dd951b2281e8883a5481a4aa79cc31a7", size = 12597324, upload-time = "2026-08-09T13:47:41.401Z" }, - { url = "https://files.pythonhosted.org/packages/33/cd/3272ba105e3bbbdaeb11357eda31e7a6825ffe159e8171665660299a948f/numpy-2.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:a610dc7e3c52edd39c2bc2375ff9c3fd59cb3ad00e4472d36f83bc1457145788", size = 10680466, upload-time = "2026-08-09T13:47:44.873Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0e/58370637b1bb70a5c9ce2b43f4b521ccb224e36ccb76a6596b17ae4b447c/numpy-2.5.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:40f4d451aed46a8046a1aae41c4e55fb3612273df9c502480135e1501576a34b", size = 16993947, upload-time = "2026-08-09T13:47:48.97Z" }, - { url = "https://files.pythonhosted.org/packages/10/93/2abcb807712b289d6d60fe4cf30532f98974a8396d885650f3ba5a13026e/numpy-2.5.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c081cbe16ba1ab53078e5ff29013621e33c509eedab055775d956427712c236e", size = 12025331, upload-time = "2026-08-09T13:47:52.646Z" }, - { url = "https://files.pythonhosted.org/packages/8b/3a/2898e003a5fbaf87e76c039b4ee1f5eb390471b4ffe74887c1f34c4e791e/numpy-2.5.2-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:0090ccdd57ec2703e9b49d0bf554767370581c1dd0a6b2bb2b2d9def317d042a", size = 5472336, upload-time = "2026-08-09T13:47:55.403Z" }, - { url = "https://files.pythonhosted.org/packages/61/a5/23f69d07c544597b29758b31b55c27dc9d541012a2c1496189fef702aec2/numpy-2.5.2-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:6a9bb119fb8dd21ba30b3f0e555b7e2b081bd9883af21ec9c1c633d161cda3a8", size = 6788387, upload-time = "2026-08-09T13:47:58.192Z" }, - { url = "https://files.pythonhosted.org/packages/15/ea/c0dbdbcf22f43782510a3e492dd3da73c6112b69cac8929d16d127536fc4/numpy-2.5.2-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a839318485284a6fb31be4f8f2c91c8f2cb22f4543c4a8903f12b0671ffe07cc", size = 15667096, upload-time = "2026-08-09T13:48:01.562Z" }, - { url = "https://files.pythonhosted.org/packages/fc/5e/29c73c31748cdb0f7566642125ba17fd5b56780cddf891b085dab27e4466/numpy-2.5.2-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba0a474801b8dc67b66bf465548abc90e82b44d2611b5770f33008dcabffe8ec", size = 16751730, upload-time = "2026-08-09T13:48:05.706Z" }, - { url = "https://files.pythonhosted.org/packages/47/95/02501e8454796bb58dadf7a99d3181e0b464bf264e1003039572f9779fac/numpy-2.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0a4035ae1129ff8777f08bfbd44f1e5d8e9c049ce0c2dd78fc0d92c13e7251c0", size = 17038686, upload-time = "2026-08-09T13:48:09.627Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b5/53a681d91b5c82687067d8ea5035e02d917b5509d6f334cb06484a954714/numpy-2.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:77843ca236b777e67f8d6b3660ea116e499612703a0ecd7093f316201eb9d8e2", size = 18507727, upload-time = "2026-08-09T13:48:13.744Z" }, - { url = "https://files.pythonhosted.org/packages/42/06/6e11443f7b64ee376c860506091103bf68f92d2cab9e8d96d4501babf07c/numpy-2.5.2-cp315-cp315t-win32.whl", hash = "sha256:7354826bc6f8f69402e9b7fe28d15fcd34feebd74f856f111585c5b0c9fb0251", size = 6269775, upload-time = "2026-08-09T13:48:17.543Z" }, - { url = "https://files.pythonhosted.org/packages/f1/18/195d6b86cd72dbbc501edfa778005fa6b87afd34c153e46028cd3a0938f4/numpy-2.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:e5651f3f87add730ee6608d915009e19c911fba0cb000c7e3ea994b7d768eb12", size = 12782559, upload-time = "2026-08-09T13:48:21.023Z" }, - { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, -] - [[package]] name = "packaging" version = "26.3" From 747f1959ffba4df0ffcc69b6e08b9df1da54d473 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 21:03:10 -0400 Subject: [PATCH 18/18] refactor(debug): run debug browser and probe on websockets debug browser: Serve the page and the /ws socket from websockets.asyncio.server instead of aiohttp, and declare websockets>=13.1,<18. The server binds 127.0.0.1 only and checks every request before the handshake: the Host must be this listener, the page and the socket need the per-run ?t= token, and the socket needs the page's own Origin (without the port on port 80, as browsers send it). Refusals are fixed text. The page is served with no-store, nosniff, frame-deny and no-referrer headers, and no Server header. The listener is bound before the URL is shown, so the URL is the one being served, now http://127.0.0.1:PORT/?t=... A frame that isn't a valid message is dropped instead of ending the connection, binary frames are ignored, and what the page can make the command keep is bounded: messages of up to 64 KiB, at most 1,000 and 8 MiB of them as stored, and only the 12 known capability names; anything else is counted and noted after the results. Text from the page is printed as escaped ASCII, never as markup. Server logs go to a silenced logger. The command's output, including the page link, now goes to stderr; under -o json or yaml, stdout holds only the result. Tests use real loopback servers instead of aiohttp mocks. debug probe: Run the probe proxy on websockets.asyncio (server and client) instead of aiohttp, and declare websockets>=13.1,<18. Frames are relayed as they arrive, so text stays text and binary stays binary, up to 4 MiB each way as before. When either side closes, the other direction is cancelled, the final probe and summary run, and the client is closed with 1000. The proxy accepts connections only from local pages and native clients. Checked before the handshake and before any upstream connection, the Host must name this listener (without the port on port 80) and an Origin, if sent, must be a page on this machine. The target must start with "/". A refused Origin prints one line. The key is only sent in the upstream Authorization header; client headers and subprotocols are not forwarded, and the client and server loggers are silenced. Text from requests is printed as escaped ASCII, and transcripts with their Unicode kept, never as markup. ffprobe snapshots run as asyncio subprocesses, so they no longer hold up other connections or Ctrl+C. On POSIX, Ctrl+C is handled on the event loop: the first stops listening, stops relaying (frames read after it are counted as dropped) and lets open connections print their summaries; a second closes them without waiting for close handshakes, stops a running ffprobe and skips the final probe. A SIGINT that was ignored when the command started stays ignored, and the previous handler is put back on exit. The upstream URL base is an attribute, so tests point it at a loopback server. With websockets 15 or later, the upstream connection follows HTTPS_PROXY and NO_PROXY, as listen and the CLI's HTTP requests do. The upstream opens through deepctl_core.websocket.open_async, so a redirect on the upgrade is still refused before any request to the new location ("Upstream connection refused", no audio sent); the aiohttp trace hook that did this goes with aiohttp. debug browser and debug probe now run on websockets, so neither needs aiohttp. Nothing else did, so it leaves the install with the seven dists only it pulled in: aiohappyeyeballs, aiosignal, async-timeout, frozenlist, multidict, propcache and yarl (about 11 MiB on linux/amd64). uv.lock is regenerated with uv 0.13.6. The eight names leave the dependency allowlist, and aiohttp joins the dists that must not come back. --- packages/deepctl-cmd-debug-browser/README.md | 2 +- .../deepctl-cmd-debug-browser/pyproject.toml | 2 +- .../src/deepctl_cmd_debug_browser/command.py | 503 +++++++---- .../static/debug.html | 4 +- .../tests/unit/test_browser_command.py | 708 ++++++++++++++-- .../tests/unit/test_websockets_floor.py | 22 + packages/deepctl-cmd-debug-probe/README.md | 2 +- .../deepctl-cmd-debug-probe/pyproject.toml | 2 +- .../src/deepctl_cmd_debug_probe/access.py | 63 ++ .../src/deepctl_cmd_debug_probe/command.py | 78 +- .../src/deepctl_cmd_debug_probe/proxy.py | 475 +++++++---- .../tests/unit/test_probe.py | 363 ++++---- .../tests/unit/test_proxy_loopback.py | 737 ++++++++++++++++ tests/unit/dependency_allowlist.txt | 12 +- tests/unit/test_dependency_allowlist.py | 8 +- uv.lock | 800 +----------------- 16 files changed, 2462 insertions(+), 1319 deletions(-) create mode 100644 packages/deepctl-cmd-debug-browser/tests/unit/test_websockets_floor.py create mode 100644 packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/access.py create mode 100644 packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py diff --git a/packages/deepctl-cmd-debug-browser/README.md b/packages/deepctl-cmd-debug-browser/README.md index 61b1710f..d31d5043 100644 --- a/packages/deepctl-cmd-debug-browser/README.md +++ b/packages/deepctl-cmd-debug-browser/README.md @@ -39,7 +39,7 @@ pipx run deepctl --help - `rich>=13.0.0,<16` - `pydantic>=2.0.0,<3` - `httpx>=0.24.0,<1` -- `aiohttp>=3.9.0` +- `websockets>=13.1,<18` ## License diff --git a/packages/deepctl-cmd-debug-browser/pyproject.toml b/packages/deepctl-cmd-debug-browser/pyproject.toml index 9be8c267..bef237b3 100644 --- a/packages/deepctl-cmd-debug-browser/pyproject.toml +++ b/packages/deepctl-cmd-debug-browser/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "rich>=13.0.0,<16", "pydantic>=2.0.0,<3", "httpx>=0.24.0,<1", - "aiohttp>=3.9.0", + "websockets>=13.1,<18", ] [project.optional-dependencies] diff --git a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py index f765fa43..781382cd 100644 --- a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py +++ b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py @@ -2,17 +2,26 @@ import importlib.resources import json +import logging +import re import socket import time import webbrowser from pathlib import Path -from typing import Any - -from deepctl_core import AuthManager, BaseCommand, Config, DeepgramClient -from rich.console import Console +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs, urlsplit + +from deepctl_core import ( + AuthManager, + BaseCommand, + Config, + DeepgramClient, + get_status_console, +) from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table +from rich.text import Text from .models import ( BrowserCapabilities, @@ -22,24 +31,111 @@ WebSocketMessage, ) -console = Console() +if TYPE_CHECKING: + from websockets.asyncio.server import Server, ServerConnection + from websockets.http11 import Request, Response + +# Everything this command prints is chrome, and the page link carries the +# session token, so all of it goes to stderr: under -o json or yaml, stdout +# holds only the result. +console = get_status_console() + +PORT_RANGE = range(3000, 3100) +MAX_FRAME_BYTES = 2**20 +# What the page can make this process keep: messages by count and by total +# size, and capability names. +MAX_MESSAGES = 1000 +MAX_MESSAGE_BYTES = 8 * 2**20 # counted as stored JSON +MAX_MESSAGE_TEXT = 64 * 2**10 # a longer frame is dropped +CAPABILITY_KEYS = ( + "web_audio_api", + "audio_context", + "audio_worklet", + "websocket_api", + "fetch_api", + "es6_features", + "dom_apis", + "console_api", + "timer_apis", + "secure_context", + "microphone_access", + "websocket_deepgram", +) +_CONTROL = re.compile(r"[\x00-\x1f\x7f]") +# The same fixed body for every 403: it echoes nothing from the request and +# doesn't say which check failed. +FORBIDDEN = "Forbidden. Open the exact link that dg debug browser printed.\n" +LINK_NOTE = "It works only while this command runs." + +# The page and the WebSocket are only for the local browser this command +# opens: loopback Host, the page's own Origin, and the session token. +PAGE_HEADERS = ( + ("Content-Type", "text/html; charset=utf-8"), + ("Cache-Control", "no-store"), + ("X-Content-Type-Options", "nosniff"), + ("X-Frame-Options", "DENY"), + ("Content-Security-Policy", "frame-ancestors 'none'"), + ("Referrer-Policy", "no-referrer"), +) + + +def _quiet_logger() -> logging.Logger: + """A logger for the server that never writes request headers anywhere, + whatever the root logger's level or handlers.""" + logger = logging.getLogger("deepctl.debug.browser.server") + if not logger.handlers: + logger.addHandler(logging.NullHandler()) + logger.propagate = False + logger.setLevel(logging.CRITICAL + 1) + return logger + + +def _untrusted(text: str, style: str = "") -> Text: + """Text from the page for rich to print as it is: plain ASCII, control + characters and anything non-ASCII as backslash escapes, in a Text object + so neither markup nor :emoji: codes in it are interpreted.""" + text = text.encode("ascii", "backslashreplace").decode("ascii") + return Text(_CONTROL.sub(lambda m: f"\\x{ord(m.group()):02x}", text), style=style) -# aiohttp and asyncio (with ssl) load only when the servers start, so -# `dg debug browser --help` doesn't import them (W3.2). +def _count(n: int, one: str, many: str) -> str: + return f"{n} {one if n == 1 else many}" + + +def _print_link(lead: str, url: str, note: str) -> None: + """The page link, whole on one line so it can be copied, then a dim note. + These lines are the only place the session token is shown (stderr).""" + console.print(f"\n{lead} [cyan]{url}[/cyan]", soft_wrap=True) + console.print(f"[dim]{note}[/dim]", soft_wrap=True) + + +def _respond( + connection: "ServerConnection", + status: int, + text: str, + headers: tuple[tuple[str, str], ...] = (), +) -> "Response": + """A complete response that ends the connection. + + connection.respond() also marks the handshake as refused, so websockets + never runs the WebSocket handler for it (13.x would for a hand-built + Response). + """ + response = connection.respond(status, text) + if headers: + del response.headers["Content-Type"] + for name, value in headers: + response.headers[name] = value + return response + + +# asyncio (with ssl) loads only when the server starts, so +# `dg debug browser --help` doesn't import it. def __getattr__(name: str) -> Any: if name == "asyncio": import asyncio return asyncio - if name == "web": - from aiohttp import web - - return web - if name == "WSMsgType": - from aiohttp import WSMsgType - - return WSMsgType raise AttributeError(f"module {__name__!r} has no attribute {name!r}") @@ -72,6 +168,12 @@ def __init__(self) -> None: self.ws_clients: set[Any] = set() self.debug_complete = False self.start_time: float | None = None + self._token = "" + self._server: Server | None = None + self.browser_opened = False + self._stored_bytes = 0 + self.dropped_messages = 0 + self.dropped_capabilities = 0 def get_arguments(self) -> list[dict[str, Any]]: """Get command arguments and options.""" @@ -111,119 +213,198 @@ def get_arguments(self) -> list[dict[str, Any]]: def find_available_port(self, start_port: int = 3000) -> int: """Find an available port starting from the given port.""" for port in range(start_port, start_port + 100): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.bind(("localhost", port)) - sock.close() + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind(("127.0.0.1", port)) + except OSError: + continue return port - except OSError: - continue raise RuntimeError(f"Could not find available port starting from {start_port}") - async def websocket_handler(self, request: Any) -> Any: - """Handle WebSocket connections from the browser.""" - from aiohttp import WSMsgType, web + @staticmethod + def _allowed(values: list[str], form: str, port: int) -> bool: + """True if there is one value, and it is `form` for 127.0.0.1 or + localhost on `port` (case-insensitive; on port 80 without the port, + as browsers send it).""" + forms = [form] + if port == 80: + forms.append(form.replace(":{port}", "")) + return len(values) == 1 and values[0].lower() in { + f.format(host=host, port=port) + for f in forms + for host in ("127.0.0.1", "localhost") + } - ws = web.WebSocketResponse() - await ws.prepare(request) - self.ws_clients.add(ws) + def _token_ok(self, target: str) -> bool: + import hmac + + tokens = parse_qs(urlsplit(target).query).get("t", []) + return len(tokens) == 1 and hmac.compare_digest( + tokens[0].encode("utf-8", "surrogateescape"), + self._token.encode(), + ) + def _process_request( + self, connection: "ServerConnection", request: "Request" + ) -> "Response | None": + """Serve the page, refuse, or let a /ws upgrade through. + + Runs before any WebSocket handshake. Rejections are fixed text and + never echo the Host, Origin or token back. + """ + # websockets < 17 closes the connection on other methods itself, + # without a response. + if getattr(request, "method", "GET") != "GET": + return _respond(connection, 405, "Method not allowed.\n") + port = connection.local_address[1] + if not self._allowed(request.headers.get_all("Host"), "{host}:{port}", port): + return _respond(connection, 403, FORBIDDEN) + path = urlsplit(request.path).path + if path not in ("/", "/ws"): + return _respond(connection, 404, "Not found.\n") + if not self._token_ok(request.path): + return _respond(connection, 403, FORBIDDEN) + if path == "/": + return _respond(connection, 200, self._page(), PAGE_HEADERS) + origins = request.headers.get_all("Origin") + if not self._allowed(origins, "http://{host}:{port}", port): + return _respond(connection, 403, FORBIDDEN) + return None + + @staticmethod + def _page() -> str: + """The debug page, read from the package.""" try: - async for msg in ws: - if msg.type == WSMsgType.TEXT: - data = json.loads(msg.data) - ws_msg = WebSocketMessage( - type=MessageType(data.get("type", "info")), - data=data.get("data", {}), - message=data.get("message"), - ) - self.messages.append(ws_msg) - - # Handle different message types - if ws_msg.type == MessageType.CAPABILITY_CHECK: - cap_data = ws_msg.data.get("result", {}) - cap_key = ws_msg.data.get("capability") - if cap_key and cap_data: - self.capabilities_data[cap_key] = BrowserCapability( - **cap_data - ) - - elif ws_msg.type == MessageType.COMPLETE: - # Store complete capabilities - if "capabilities" in ws_msg.data: - caps = ws_msg.data["capabilities"] - # Convert all capability data - cap_dict = {} - for key in [ - "web_audio_api", - "audio_context", - "audio_worklet", - "websocket_api", - "fetch_api", - "es6_features", - "dom_apis", - "console_api", - "timer_apis", - "secure_context", - "microphone_access", - "websocket_deepgram", - ]: - if key in caps: - cap_dict[key] = BrowserCapability(**caps[key]) - elif key in self.capabilities_data: - cap_dict[key] = self.capabilities_data[key] - - # Store complete capabilities - self.capabilities_data = { - **cap_dict, - "user_agent": caps.get("user_agent", "Unknown"), - "overall_compatible": caps.get( - "overall_compatible", False - ), - } - self.debug_complete = True - elif msg.type == WSMsgType.ERROR: - console.print(f"[red]WebSocket error: {ws.exception()}[/red]") + files = importlib.resources.files("deepctl_cmd_debug_browser") + html_content = (files / "static" / "debug.html").read_text() + except Exception: + # Fallback to Path-based approach + html_path = Path(__file__).parent / "static" / "debug.html" + html_content = html_path.read_text() + return html_content + + def _handle_text(self, text: str) -> None: + """Record one message from the page. + + Raises ValueError, TypeError or AttributeError for a frame that isn't + a valid message; nothing is recorded then. + """ + if len(text) > MAX_MESSAGE_TEXT: + self.dropped_messages += 1 + return + data = json.loads(text) + ws_msg = WebSocketMessage( + type=MessageType(data.get("type", "info")), + data=data.get("data", {}), + message=data.get("message"), + ) + capabilities = self.capabilities_data + + # Handle different message types + if ws_msg.type == MessageType.CAPABILITY_CHECK: + cap_data = ws_msg.data.get("result", {}) + cap_key = ws_msg.data.get("capability") + if cap_key and cap_key not in CAPABILITY_KEYS: + self.dropped_capabilities += 1 + elif cap_key and cap_data: + capabilities = { + **capabilities, + cap_key: BrowserCapability(**cap_data), + } + + elif ws_msg.type == MessageType.COMPLETE: + # Store complete capabilities + if "capabilities" in ws_msg.data: + caps = ws_msg.data["capabilities"] + # Convert all capability data + cap_dict = {} + for key in CAPABILITY_KEYS: + if key in caps: + cap_dict[key] = BrowserCapability(**caps[key]) + elif key in self.capabilities_data: + cap_dict[key] = self.capabilities_data[key] + + # Store complete capabilities + capabilities = { + **cap_dict, + "user_agent": caps.get("user_agent", "Unknown"), + "overall_compatible": caps.get("overall_compatible", False), + } + + size = len(ws_msg.model_dump_json()) + if ( + len(self.messages) < MAX_MESSAGES + and self._stored_bytes + size <= MAX_MESSAGE_BYTES + ): + self.messages.append(ws_msg) + self._stored_bytes += size + else: + self.dropped_messages += 1 + self.capabilities_data = capabilities + if ws_msg.type == MessageType.COMPLETE: + self.debug_complete = True + + async def websocket_handler(self, ws: "ServerConnection") -> None: + """Handle WebSocket connections from the browser. + + Text frames carry JSON messages; a frame that isn't a valid message + is dropped and the connection stays open. Binary frames are ignored. + """ + from websockets.exceptions import ConnectionClosed + + self.ws_clients.add(ws) + try: + async for frame in ws: + if isinstance(frame, str): + try: + self._handle_text(frame) + except (ValueError, TypeError, AttributeError): + continue + except ConnectionClosed: + pass finally: self.ws_clients.discard(ws) - await ws.close() - return ws + async def _start_server(self, port: int | None) -> int: + """Listen on 127.0.0.1 and return the port. - async def http_handler(self, request: Any) -> Any: - """Serve the debug HTML page.""" - from aiohttp import web + A given port is tried once and an OSError propagates (0 picks any + free port). Otherwise the first port in 3000-3099 that binds is used. + """ + from websockets.asyncio.server import serve - if request.path == "/": - # Read the HTML template from the static directory + for candidate in PORT_RANGE if port is None else [port]: try: - files = importlib.resources.files("deepctl_cmd_debug_browser") - html_content = (files / "static" / "debug.html").read_text() - except Exception: - # Fallback to Path-based approach - html_path = Path(__file__).parent / "static" / "debug.html" - html_content = html_path.read_text() + self._server = await serve( + self.websocket_handler, + "127.0.0.1", + candidate, + process_request=self._process_request, + max_size=MAX_FRAME_BYTES, + ping_interval=None, + server_header=None, + logger=_quiet_logger(), + ) + except OSError: + if port is not None: + raise + continue + bound: int = next(iter(self._server.sockets)).getsockname()[1] + return bound + raise RuntimeError( + f"Could not find available port starting from {PORT_RANGE.start}" + ) - return web.Response(text=html_content, content_type="text/html") - return web.Response(status=404) + async def _stop_server(self) -> None: + if self._server is not None: + self._server.close() + await self._server.wait_closed() + self._server = None async def run_servers(self, port: int, timeout: int) -> dict[str, Any]: - """Run both HTTP and WebSocket servers.""" + """Wait for the page to finish its checks, or for the timeout.""" import asyncio - from aiohttp import web - - # Create aiohttp app - app = web.Application() - app.router.add_get("/", self.http_handler) - app.router.add_get("/ws", self.websocket_handler) - - # Start HTTP server (which also handles WebSocket) - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, "localhost", port) - await site.start() - console.print( f"\n[green]✓[/green] Debug server started on port [cyan]{port}[/cyan]" ) @@ -233,9 +414,6 @@ async def run_servers(self, port: int, timeout: int) -> dict[str, Any]: while not self.debug_complete and (time.time() - start_time) < timeout: await asyncio.sleep(0.1) - # Cleanup - await runner.cleanup() - return { "completed": self.debug_complete, "timed_out": not self.debug_complete @@ -274,7 +452,7 @@ def display_results( log_table.add_row( time_str, f"[{type_style}]{msg.type.value}[/{type_style}]", - msg.message or "", + _untrusted(msg.message or ""), ) console.print(log_table) @@ -314,14 +492,21 @@ def display_results( status_color = "green" if cap.supported else "red" cap_table.add_row( display_name, - f"[{status_color}]{status_icon} {cap.details}[/{status_color}]", - cap.version or "", + Text.assemble( + f"{status_icon} ", + _untrusted(str(cap.details)), + style=status_color, + ), + _untrusted(cap.version or ""), ) console.print(cap_table) # Overall compatibility - console.print(f"\n[bold]User Agent:[/bold] {capabilities.user_agent}") + console.print( + Text("\nUser Agent:", style="bold"), + _untrusted(capabilities.user_agent), + ) if capabilities.overall_compatible: console.print( "[bold green]✓ Browser is fully compatible with " @@ -358,6 +543,7 @@ def handle( ) -> Any: """Handle browser debug command execution.""" import asyncio + import secrets port = kwargs.get("port") no_browser = kwargs.get("no_browser", False) @@ -365,10 +551,7 @@ def handle( save_report = kwargs.get("save_report") self.start_time = time.time() - - # Find available port - if not port: - port = self.find_available_port() + self._token = secrets.token_urlsafe(16) # Show initial message console.print( @@ -380,7 +563,23 @@ def handle( ) ) - url = f"http://localhost:{port}" + # Listen first, so the URL shown is the one that is being served. + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + port = loop.run_until_complete(self._start_server(port or None)) + result = self._serve(loop, port, timeout, no_browser) + finally: + loop.run_until_complete(self._stop_server()) + loop.close() + + return self._finish(port, timeout, save_report, result) + + def _serve( + self, loop: Any, port: int, timeout: int, no_browser: bool + ) -> dict[str, Any]: + """Point the browser at the page, then wait for its checks.""" + url = f"http://127.0.0.1:{port}/?t={self._token}" # Prompt user to open browser if not no_browser: @@ -389,18 +588,18 @@ def handle( "\n[yellow]Press Enter to open the debugger in your browser...[/yellow]" ) input() - else: - console.print( - f"\n[dim]Opening [cyan]{url}[/cyan] in your browser...[/dim]" - ) + _print_link( + "Opening the debugger in your browser:", + url, + f"If it doesn't open, paste the link into your browser. {LINK_NOTE}", + ) webbrowser.open(url) - browser_opened = True + self.browser_opened = True else: - console.print( - f"\n[dim]Open [cyan]{url}[/cyan] in your browser to start " - f"debugging[/dim]" + _print_link( + "Open this link in your browser to start debugging:", url, LINK_NOTE ) - browser_opened = False + self.browser_opened = False # Run the async event loop with Progress( @@ -410,18 +609,20 @@ def handle( transient=True, ) as progress: task = progress.add_task("Waiting for browser connection...", total=None) - - # Run servers - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - try: - result = loop.run_until_complete(self.run_servers(port, timeout)) - finally: - loop.close() - + result: dict[str, Any] = loop.run_until_complete( + self.run_servers(port, timeout) + ) progress.update(task, completed=True) + return result + def _finish( + self, + port: int, + timeout: int, + save_report: str | None, + result: dict[str, Any], + ) -> BrowserDebugResult: + """Show the results and build the command result.""" # Process results capabilities = None if self.capabilities_data and "overall_compatible" in self.capabilities_data: @@ -462,6 +663,22 @@ def handle( console.print("[dim]No browser connection was established.[/dim]") else: console.print("\n[yellow]Debug session ended[/yellow]") + left_out = [] + if self.dropped_messages: + left_out.append( + _count(self.dropped_messages, "message", "messages") + + " over the size limit" + ) + if self.dropped_capabilities: + left_out.append( + _count( + self.dropped_capabilities, + "unknown capability name", + "unknown capability names", + ) + ) + if left_out: + console.print(f"[dim]Left out of the results: {', '.join(left_out)}[/dim]") # Build result errors = [ @@ -483,7 +700,7 @@ def handle( errors=errors, warnings=warnings, duration_seconds=result["duration"], - browser_opened=browser_opened, + browser_opened=self.browser_opened, ) # Save report if requested diff --git a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html index b5bf50f0..12b7ece1 100644 --- a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html +++ b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html @@ -232,8 +232,8 @@

Debug Complete!