From c11d8116cd4c7f4980a98380f29db1a363102089 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:34:54 -0400 Subject: [PATCH 1/8] feat(core): resolve credentials in one place deepctl_core.credentials.resolve() decides the API key and project ID for a run and says where each came from (key_source, profile_source, untrusted_source). The AuthManager methods (get_api_key, get_project_id, get_credential_source, is_authenticated, has_profile_credentials) answer from it, once per run, and dg whoami takes its key source from it. Precedence is 0.4.1's, unchanged. Because Config merges DEEPGRAM_API_KEY into the active default profile, that is --api-key > keyring > env > file for the default profile, and --api-key > keyring > file > env for another profile. User-visible changes: - With no API key, commands return the coded auth_required error: a coded result on stdout under a machine format and two stderr lines (ERROR and HINT). The error text "DEEPGRAM_API_KEY is not set" is unchanged; the old two-line "not set in the configuration file ... Run deepctl login" message is replaced by the catalogue hint. When the only key was in a user config file an AI agent session doesn't trust, the error is E-KEY-UNTRUSTED with next_step null. - A DEEPGRAM_API_KEY, DEEPGRAM_PROFILE, DEEPGRAM_PROJECT_ID or DEEPGRAM_BASE_URL that still holds an unexpanded ${...} placeholder is ignored, with one W-ENV-PLACEHOLDER warning naming the variable. - Off a terminal (including MCP), a keyring read gets 2 s; when the keyring doesn't answer, dg uses the config file or the environment instead and prints one W-KEYRING-TIMEOUT warning. This covers dg whoami and dg profiles too. With --api-key the keyring isn't read or imported. - "Using credentials from environment variables" now also appears when DEEPGRAM_API_KEY is the key in use; "No project ID specified" appears only for commands that need a project. - Commands that use the key remove an expired credential's keyring entry, as in 0.4.1, but only when the expiry comes from a trusted user file. whoami, login's checks, is_authenticated() and has_profile_credentials() stay read-only, as in 0.4.1. - auth_invalid's details.key_source comes from the same resolution, so the error path reads the keyring no second time. - AuthManager builds its HTTP client on first use, and group commands get a lazy client, so `dg debug --help` loads no httpx or keyring. The env-vs-stored mismatch check is implemented behind a constant that stays off until login writes fingerprints. --- conftest.py | 11 + .../src/deepctl_cmd_login/command.py | 55 +- .../tests/unit/test_keyring_bound.py | 146 ++++ .../tests/unit/test_login_command.py | 12 +- .../deepctl-core/src/deepctl_core/auth.py | 396 ++++++----- .../src/deepctl_core/base_command.py | 74 +- .../src/deepctl_core/base_group_command.py | 34 +- .../src/deepctl_core/credentials.py | 541 +++++++++++++++ .../src/deepctl_core/transport.py | 76 ++- packages/deepctl-core/tests/unit/test_auth.py | 5 +- .../tests/unit/test_auth_insecure.py | 3 +- .../tests/unit/test_config_location.py | 41 +- .../tests/unit/test_credentials.py | 630 ++++++++++++++++++ .../tests/unit/test_output_channels.py | 3 +- src/deepctl/_cli.py | 6 + tests/unit/test_command_imports.py | 14 +- 16 files changed, 1741 insertions(+), 306 deletions(-) create mode 100644 packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py create mode 100644 packages/deepctl-core/src/deepctl_core/credentials.py create mode 100644 packages/deepctl-core/tests/unit/test_credentials.py diff --git a/conftest.py b/conftest.py index 86d564bc..5ae8166f 100644 --- a/conftest.py +++ b/conftest.py @@ -88,3 +88,14 @@ def _restore_sigint(): yield if before is not None: signal.signal(signal.SIGINT, before) + + +@pytest.fixture(autouse=True) +def _fresh_credentials_state(): + """Forget per-process resolver state (placeholder warnings, a keyring + timeout) between tests, as each dg run starts without it.""" + from deepctl_core import credentials + + credentials._reset_for_tests() + yield + credentials._reset_for_tests() diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index 32a369e5..fe486e01 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1531,12 +1531,10 @@ def handle( # Try to get API key from keyring to verify access try: - import keyring - from deepctl_core.auth import KEYRING_SERVICE + from deepctl_core import credentials - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{switch_profile}" - ) + # Bounded off a terminal (2 s); no answer counts as no entry. + api_key = credentials.keyring_get(f"api-key.{switch_profile}") if not api_key: # Check if API key is in config instead @@ -1625,48 +1623,23 @@ def handle( **kwargs: Any, ) -> Any: """Handle whoami command.""" - import os - - import keyring as _keyring - from deepctl_core.auth import KEYRING_SERVICE - if kwargs.get("profile"): config.profile = kwargs["profile"] profile_name = config.profile or "default" profile_cfg = config.get_profile(profile_name) - # Determine key and its exact storage source. Expired finite profile - # credentials must not mask a valid environment variable. - api_key: str | None = None - key_source = "not set" - env_key = os.environ.get("DEEPGRAM_API_KEY") - has_profile_key, _ = auth_manager.has_profile_credentials(profile_name) - - if has_profile_key: - try: - api_key = _keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) - if api_key: - key_source = "system keyring" - except Exception: - pass - - if has_profile_key and not api_key and profile_cfg.api_key: - # Config merges DEEPGRAM_API_KEY into the profile at load time - # (environment overrides the file), so a profile key equal to the - # environment value came from the environment, not the config - # file — label it accordingly instead of claiming "config file". - api_key = profile_cfg.api_key - if env_key and profile_cfg.api_key == env_key: - key_source = "DEEPGRAM_API_KEY (env)" - else: - key_source = "config file" - - if not api_key and env_key: - api_key = env_key - key_source = "DEEPGRAM_API_KEY (env)" + # The key and its exact storage source, from the one resolver (its + # key_source uses these same words): one bounded keyring read, and an + # expired finite profile credential doesn't mask the environment's. + # Like before, --api-key isn't a storage source here. Read-only, as in + # 0.4.2: an expired credential's keyring entry stays (prune=False). + resolved = auth_manager.resolve( + profile=profile_name, use_flags=False, prune=False + ) + api_key = resolved.api_key + key_source = resolved.key_source + has_profile_key = resolved.profile_has_key authenticated = auth_manager.is_authenticated() masked: str | None = None diff --git a/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py b/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py new file mode 100644 index 00000000..790c1ddb --- /dev/null +++ b/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py @@ -0,0 +1,146 @@ +"""Whoami, profiles --switch and profiles --list read the keyring +through the resolver's bound (2 s off a terminal), not directly.""" + +from __future__ import annotations + +import os +import sys +import threading +import time +from unittest.mock import MagicMock + +import pytest +import yaml +from deepctl_core import auth as auth_module +from deepctl_core import config as config_module +from deepctl_core import credentials, hosts, output +from deepctl_core.auth import AuthManager +from deepctl_core.config import Config + +pytestmark = pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="XDG layout (Linux)" +) + + +class HangingKeyring: + def __init__(self) -> None: + self.release = threading.Event() + self.reads = 0 + + def get_password(self, service: str, username: str) -> str | None: + self.reads += 1 + self.release.wait(10) + return "k" * 40 + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + monkeypatch.setattr(hosts, "_reported", set()) + home = tmp_path / "home" + (home / ".config" / "deepctl").mkdir(parents=True) + cfg = home / ".config" / "deepctl" / "config.yaml" + cfg.write_text( + yaml.safe_dump( + {"profiles": {"default": {"project_id": "p"}, "work": {"api_key": "w" * 40}}} + ) + ) + os.chmod(cfg, 0o600) + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setattr(credentials, "KEYRING_BOUND_S", 0.3) + monkeypatch.setattr(credentials, "_interactive", lambda: False) + kr = HangingKeyring() + monkeypatch.setattr(auth_module, "keyring", kr, raising=False) + yield kr + kr.release.set() + + +def _timed(fn): + started = time.monotonic() + result = fn() + return result, time.monotonic() - started + + +def test_whoami_is_bounded(setup, capsys): + from deepctl_cmd_login.command import WhoamiCommand + + config = Config() + result, took = _timed( + lambda: WhoamiCommand().handle(config, AuthManager(config), MagicMock()) + ) + assert took < 5.0 # the keyring hangs until the test ends + assert result.key_source == "DEEPGRAM_API_KEY (env)" + assert result.authenticated is True + assert setup.reads == 1 # one bounded read, then cached + assert capsys.readouterr().err.count("keyring didn't answer") == 1 + + +def test_profiles_switch_is_bounded(setup, monkeypatch): + from deepctl_cmd_login.command import ProfilesCommand + + config = Config() + monkeypatch.setattr(config, "save", lambda *a, **k: None) + result, took = _timed( + lambda: ProfilesCommand().handle( + config, AuthManager(config), MagicMock(), switch="work" + ) + ) + assert took < 5.0 # the keyring hangs until the test ends + assert result.status != "error" # fell back to the profile's file key + + +def test_profiles_list_is_bounded(setup): + config = Config() + result, took = _timed(lambda: AuthManager(config).list_profiles()) + assert took < 5.0 # the keyring hangs until the test ends + # A timeout falls back to the file, like a keyring error did in 0.4.2. + assert result.profiles["work"].api_key == "****" + "w" * 4 + + +def test_whoami_leaves_an_expired_entry_alone(tmp_path, monkeypatch): + """Whoami is read-only, as in 0.4.2; key_source unchanged.""" + from deepctl_cmd_login.command import WhoamiCommand + + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + home = tmp_path / "home" + (home / ".config" / "deepctl").mkdir(parents=True) + cfg = home / ".config" / "deepctl" / "config.yaml" + cfg.write_text( + yaml.safe_dump( + {"profiles": {"default": {"dg_token_expires_at": "2020-01-01T00:00:00+00:00"}}} + ) + ) + os.chmod(cfg, 0o600) + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + + class Recording: + def __init__(self) -> None: + self.entries = {"api-key.default": "k" * 40} + self.calls: list[str] = [] + + def get_password(self, service, username): + self.calls.append("get") + return self.entries.get(username) + + def delete_password(self, service, username): + self.calls.append("delete") + self.entries.pop(username, None) + + kr = Recording() + monkeypatch.setattr(auth_module, "keyring", kr, raising=False) + config = Config() + result = WhoamiCommand().handle(config, AuthManager(config), MagicMock()) + assert result.key_source == "DEEPGRAM_API_KEY (env)" + assert "delete" not in kr.calls + assert kr.entries == {"api-key.default": "k" * 40} diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_command.py b/packages/deepctl-cmd-login/tests/unit/test_login_command.py index e63fa16d..f487a58e 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_command.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_command.py @@ -552,16 +552,18 @@ def whoami_command(self): return WhoamiCommand() def _run(self, whoami_command, mock_config, mock_client, env, profile_key): - auth_manager = Mock(spec=AuthManager) - auth_manager.get_project_id.return_value = "proj-1" - auth_manager.has_profile_credentials.return_value = (bool(profile_key), False) - auth_manager.is_authenticated.return_value = bool(profile_key or env) + # Whoami takes the key and its source from the resolver, so + # this runs the real AuthManager over the mocked config and keyring. mock_config.get_profile.return_value.api_key = profile_key + mock_config.get_profile.return_value.project_id = "proj-1" mock_config.get_profile.return_value.dg_token_expires_at = None + keyring = Mock() + keyring.get_password.return_value = None with ( patch.dict("os.environ", env, clear=False), - patch("keyring.get_password", return_value=None), + patch("deepctl_core.auth.keyring", keyring, create=True), ): + auth_manager = AuthManager(mock_config) if "DEEPGRAM_API_KEY" not in env: import os diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index 03044105..2463a8d6 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -3,14 +3,15 @@ import os import time import webbrowser +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar, cast from urllib.parse import urlsplit from pydantic import BaseModel, field_validator -from . import hosts, output +from . import credentials, hosts, output, transport from . import messages as msg from .client import _split_base_url from .config import Config @@ -24,12 +25,16 @@ import httpx # noqa: TC004 import keyring # noqa: TC004 -# httpx and keyring load when the first AuthManager is built, not when this -# module is imported (every command module imports it). _load_modules() -# binds them as module globals; __getattr__ makes `deepctl_core.auth.keyring` -# the real module before that, so tests can patch it as before. +# httpx and keyring load when they're first used, not when this module is +# imported (every command module imports it) or an AuthManager is built +# (every command builds one). __getattr__ binds them as module globals on +# first access, so `deepctl_core.auth.keyring` is the real module and tests +# can patch it as before. The keyring is read through +# deepctl_core.credentials, which bounds the read off a terminal. _LAZY_MODULES = ("httpx", "keyring") +_F = TypeVar("_F", bound=Callable[..., Any]) + def _bind_module(name: str) -> Any: import importlib @@ -48,8 +53,8 @@ def __getattr__(name: str) -> Any: return _bind_module(name) -def _load_modules() -> None: - for name in _LAZY_MODULES: +def _load_modules(*names: str) -> None: + for name in names or _LAZY_MODULES: if name not in globals(): _bind_module(name) @@ -205,6 +210,54 @@ def __init__(self, host: str) -> None: self.host = host +def _forgets_resolved(method: _F) -> _F: + """Forget cached credential answers once ``method`` has written the keyring + or a profile (also when it fails part-way), so later questions see the + new state.""" + import functools + + @functools.wraps(method) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + try: + return method(self, *args, **kwargs) + finally: + self._forget_resolved() + + return cast("_F", wrapper) + + +class AuthRequiredError(transport.TransportError, AuthenticationError): + """``auth_required``: no API key is set (``E-AUTH-REQUIRED``), or the only + one was in a user config file this AI agent session doesn't trust + (``E-KEY-UNTRUSTED``). + + A :class:`~deepctl_core.transport.TransportError`, so ``BaseCommand`` + renders the coded result, and an :class:`AuthenticationError`, so code + that caught ``guard()``'s error before 0.5.0 still does. + """ + + def __init__(self, untrusted_source: str | None = None) -> None: + if untrusted_source is None: + super().__init__("auth_required", {}, {}) + else: + super().__init__( + "auth_required", + {"user_config": untrusted_source}, + {}, + transport.KEY_UNTRUSTED, + ) + + +class _NoFlags: + """A credentials context without ``--api-key`` and ``--project-id``.""" + + explicit_api_key = None + explicit_project_id = None + + def __init__(self, config: Config) -> None: + self.config = config + + class AuthManager: """Cross-platform authentication manager.""" @@ -221,23 +274,103 @@ def __init__( explicit_api_key: Explicitly provided API key (e.g., from CLI flags) explicit_project_id: Explicitly provided project ID (e.g., from CLI flags) """ - _load_modules() self.config = config self.explicit_api_key = explicit_api_key self.explicit_project_id = explicit_project_id - # DEEPGRAM_CLI_INSECURE=1 skips TLS verification for localhost, - # 127.0.0.1 and [::1] (local dev), never for any other host and never - # in an AI agent session: the agent may have set it. Proxy settings - # from the environment still apply to every other host. - if _insecure_allowed(): - self.client = httpx.Client( - timeout=30.0, - verify=True, - mounts=_insecure_mounts(), - event_hooks={"request": [_warn_if_verified]}, + # One credential resolution per question set (deepctl_core.credentials); + # forgotten whenever this manager writes the keyring or a profile. + self._resolved: dict[tuple[Any, ...], credentials.Resolved] = {} + # Keyring entries this manager already removed (expired credentials). + self._pruned: set[str] = set() + self._client: httpx.Client | None = None + + @property + def client(self) -> "httpx.Client": + """The HTTP client for login's device flow and key verification. + + Built on first use, so no other command makes a second client; API + requests use the shared one (``deepctl_core.transport``). + """ + if self._client is None: + _load_modules("httpx") + # DEEPGRAM_CLI_INSECURE=1 skips TLS verification for localhost, + # 127.0.0.1 and [::1] (local dev), never for any other host and + # never in an AI agent session: the agent may have set it. Proxy + # settings from the environment still apply to every other host. + if _insecure_allowed(): + self._client = httpx.Client( + timeout=30.0, + verify=True, + mounts=_insecure_mounts(), + event_hooks={"request": [_warn_if_verified]}, + ) + else: + self._client = httpx.Client(timeout=30.0, verify=True) + return self._client + + @client.setter + def client(self, value: "httpx.Client | None") -> None: + self._client = value + + @client.deleter + def client(self) -> None: + self._client = None + + def resolve( + self, + *, + profile: str | None = None, + ignore_env: bool = False, + need_key: bool = True, + use_flags: bool = True, + prune: bool = True, + ) -> credentials.Resolved: + """This run's credentials and their sources (``credentials.resolve``). + + ``use_flags=False`` answers as if no ``--api-key`` or + ``--project-id`` had been given (the stored-profile questions). + ``prune=False`` keeps it read-only (introspection: whoami, login's + checks); answers are cached either way, so one run resolves once. + """ + ctx: Any = self if use_flags else _NoFlags(self.config) + name = profile or credentials.profile_name(self.config)[0] + stored = self.config.get_profile(name) + state = ( + name, + ignore_env, + ctx.explicit_api_key, + ctx.explicit_project_id, + stored.api_key, + stored.project_id, + stored.dg_token_expires_at, + ) + # A full answer also answers the project-only question. + found = self._resolved.get((True, *state)) + if found is None and not need_key: + found = self._resolved.get((False, *state)) + if found is None: + found = credentials.resolve( + ctx, + profile=profile, + ignore_env=ignore_env, + need_key=need_key, + prune=False, ) - else: - self.client = httpx.Client(timeout=30.0, verify=True) + self._resolved[(need_key, *state)] = found + # Project-only questions (need_key=False) never touch the keyring. + if ( + prune + and need_key + and found.prune_target + and found.prune_target not in self._pruned + ): + self._pruned.add(found.prune_target) + credentials.prune_expired(found) + return found + + def _forget_resolved(self) -> None: + self._resolved.clear() + self._pruned.clear() def check_base_url(self) -> str: """Return the base URL credentials may be sent to. @@ -253,8 +386,8 @@ def has_env_credentials(self) -> tuple[bool, bool]: Returns: Tuple of (has_api_key, has_project_id) """ - return bool(os.getenv("DEEPGRAM_API_KEY")), bool( - os.getenv("DEEPGRAM_PROJECT_ID") + return bool(credentials.env_value("DEEPGRAM_API_KEY")), bool( + credentials.env_value("DEEPGRAM_PROJECT_ID") ) def has_profile_credentials( @@ -262,35 +395,17 @@ def has_profile_credentials( ) -> tuple[bool, bool]: """Check if profile has stored credentials. + As in 0.4.2, a profile key is a usable keyring entry or a key in the + profile as the config holds it (which includes ``DEEPGRAM_API_KEY``). + Args: profile_name: Profile to check (uses current profile if not specified) Returns: Tuple of (has_api_key, has_project_id) """ - profile_name = profile_name or self.config.profile or "default" - - # Check keyring for a direct API key. - has_api_key = False - expired, _ = self._credential_expiry(profile_name) - if not expired: - try: - has_api_key = bool( - keyring.get_password(KEYRING_SERVICE, f"api-key.{profile_name}") - ) - except Exception: - pass - - # If not in keyring, check config. - if not has_api_key: - profile = self.config.get_profile(profile_name) - has_api_key = bool(profile.api_key) - - # Check for project ID in profile - profile = self.config.get_profile(profile_name) - has_project_id = bool(profile.project_id) - - return has_api_key, has_project_id + resolved = self.resolve(profile=profile_name, use_flags=False, prune=False) + return resolved.profile_has_key, resolved.profile_has_project def is_authenticated(self, check_profile_only: bool = False) -> bool: """Check if user is authenticated. @@ -298,24 +413,23 @@ def is_authenticated(self, check_profile_only: bool = False) -> bool: Args: check_profile_only: If True, only check profile credentials, not env vars """ - # Check explicit credentials first (highest priority) if self.explicit_api_key: return True - - # Check profile credentials - profile_name = self.config.profile or "default" - has_profile_key, _ = self.has_profile_credentials(profile_name) - if has_profile_key: + resolved = self.resolve(use_flags=False, prune=False) + if resolved.profile_has_key: return True - - # Check environment variables (unless checking profile only) - return not check_profile_only and bool(os.getenv("DEEPGRAM_API_KEY")) + return not check_profile_only and bool( + credentials.env_value("DEEPGRAM_API_KEY") + ) def is_ci_mode(self) -> bool: """Check if running in CI mode (credentials from environment).""" # If both API key and project ID are provided via environment, # we're in CI mode - return bool(os.getenv("DEEPGRAM_API_KEY") and os.getenv("DEEPGRAM_PROJECT_ID")) + return bool( + credentials.env_value("DEEPGRAM_API_KEY") + and credentials.env_value("DEEPGRAM_PROJECT_ID") + ) def get_api_key(self, ignore_env: bool = False) -> str | None: """Get API key following precedence: explicit > profile > env. @@ -327,74 +441,20 @@ def get_api_key(self, ignore_env: bool = False) -> str | None: Args: ignore_env: If True, don't check environment variables """ - # Explicit credentials (--api-key flag, CI/CD) bypass the JWT flow. - if self.explicit_api_key: - return self.explicit_api_key - - profile_name = self.config.profile or "default" - current_profile = self.config.get_profile(profile_name) - - expired, expiry_error = self._credential_expiry(profile_name) - if not expired: - # Profile keyring entry, followed by its config-file fallback. - try: - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) - if api_key: - return api_key - except Exception: - pass - - if current_profile.api_key: - return current_profile.api_key - else: - self._delete_keyring_credential(f"api-key.{profile_name}") - - # Environment variable — lowest priority. - if not ignore_env: - api_key = os.getenv("DEEPGRAM_API_KEY") - if api_key: - return api_key - - if expiry_error: - raise AuthenticationError(expiry_error) - + resolved = self.resolve(ignore_env=ignore_env) + if resolved.api_key: + return resolved.api_key + if resolved.expiry_error: + raise AuthenticationError(resolved.expiry_error) return None def _credential_expiry(self, profile_name: str) -> tuple[bool, str | None]: """Return whether a finite profile credential is unusable and why.""" - expires_at = self.config.get_profile(profile_name).dg_token_expires_at - if not isinstance(expires_at, str): - return False, None - - try: - parsed_expiry = datetime.fromisoformat(expires_at) - if parsed_expiry.tzinfo is None: - raise ValueError - except (TypeError, ValueError): - return ( - True, - "Your Deepgram credential expiry is invalid. Run `dg login` again.", - ) - - if parsed_expiry <= datetime.now(timezone.utc): - return True, "Your Deepgram credential has expired. Run `dg login` again." - - return False, None + return credentials.credential_expiry(self.config, profile_name) def _delete_keyring_credential(self, keyring_key: str) -> None: """Remove a credential when present without masking keyring failures.""" - from keyring.errors import PasswordDeleteError - - try: - keyring.delete_password(KEYRING_SERVICE, keyring_key) - except PasswordDeleteError: - pass - except Exception as exc: - console.print( - f"[yellow]Warning:[/yellow] Could not remove legacy credential: {exc}" - ) + credentials.keyring_delete(keyring_key) def get_project_id(self, ignore_env: bool = False) -> str | None: """Get project ID following precedence: explicit > profile > env. @@ -402,23 +462,7 @@ def get_project_id(self, ignore_env: bool = False) -> str | None: Args: ignore_env: If True, don't check environment variables """ - # Explicit credentials have highest priority - if self.explicit_project_id: - return self.explicit_project_id - - # Check profile next - profile_name = self.config.profile or "default" - current_profile = self.config.get_profile(profile_name) - if current_profile.project_id: - return current_profile.project_id - - # Environment variable has lowest priority - if not ignore_env: - project_id = os.getenv("DEEPGRAM_PROJECT_ID") - if project_id: - return project_id - - return None + return self.resolve(ignore_env=ignore_env, need_key=False).project_id def get_credential_source(self) -> str: """Get a description of where credentials are coming from. @@ -429,16 +473,18 @@ def get_credential_source(self) -> str: if self.explicit_api_key or self.explicit_project_id: return "explicit flags" - profile_name = self.config.profile or "default" - has_profile_key, has_profile_project = self.has_profile_credentials( - profile_name - ) - - if has_profile_key or has_profile_project: - return f"profile '{profile_name}'" - - has_env_key, has_env_project = self.has_env_credentials() - if has_env_key or has_env_project: + # The real source: a profile key equal to DEEPGRAM_API_KEY came from + # the environment (Config merges it into the profile). + resolved = self.resolve() + if resolved.key_source in ( + credentials.KEY_SOURCE_KEYRING, + credentials.KEY_SOURCE_FILE, + ) or (resolved.project_source == "profile"): + return f"profile '{resolved.profile}'" + if ( + resolved.key_source == credentials.KEY_SOURCE_ENV + or resolved.project_source == "env" + ): return "environment variables" return "no credentials" @@ -454,6 +500,7 @@ def verify_api_key( Returns: Tuple of (success, message, error_type) """ + _load_modules("httpx") if not api_key: api_key = self.get_api_key() @@ -520,6 +567,7 @@ def verify_credentials( - error_type: 'auth' for API key issues, 'project' for project ID issues, None if successful """ + _load_modules("httpx") # Use provided credentials or get from storage if not api_key: api_key = self.get_api_key() @@ -582,38 +630,26 @@ def guard(self) -> None: """Check that an API key is available, without sending it anywhere. Refuses first, before reading any key, when the base URL isn't - trusted with credentials. The key itself is checked by the command's - own request: a 401 or 403 there becomes ``auth_invalid`` or - ``auth_forbidden`` (see ``deepctl_core.transport``). Project ID - validation is handled separately by requires_project on BaseCommand. + trusted with credentials. With no key, raises :class:`AuthRequiredError` + (``auth_required``), which ``BaseCommand`` renders; the key itself + is checked by the command's own request: a 401 or 403 there becomes + ``auth_invalid`` or ``auth_forbidden`` (see ``deepctl_core.transport``). + Project ID validation is handled separately by requires_project on + BaseCommand. """ self.check_base_url() - api_key = self.get_api_key() - - if not api_key and getattr(self.config, "api_key_untrusted", False) is True: + # Raises the expiry error when a stored credential expired and + # nothing else is set, as before. + if self.get_api_key(): + return + resolved = self.resolve() + if resolved.untrusted_source: # The file's key was left unused (W-PERM-AGENT / W-CFG-UNTRUSTED): # say so, and send the agent to a person, not to dg login. - from rich.markup import escape - - text = msg.E_KEY_UNTRUSTED.format( - user_config=msg.display_path(self.config.config_path) - ) - output.print_error(escape(text)) - output.print_hint(msg.W_CFG_UNTRUSTED_FILE_HINT) - raise AuthenticationError(text) - if not api_key: - console.print( - "[red]Error:[/red] DEEPGRAM_API_KEY is not set in the " - "configuration file " - f"({self.config.config_path}) or environment variable.\n" - ) - console.print( - "[yellow]Run[/yellow] [bold]deepctl login[/bold] " - "[yellow]to configure the CLI with your Deepgram " - "account.[/yellow]\n" - ) - raise AuthenticationError("DEEPGRAM_API_KEY is not set") + raise AuthRequiredError(untrusted_source=resolved.untrusted_source) + raise AuthRequiredError() + @_forgets_resolved def login_with_api_key( self, api_key: str, project_id: str | None = None, _force_write: bool = False ) -> None: @@ -651,6 +687,7 @@ def login_with_api_key( console.print(f"[green]✓[/green] {message}") # Store API key in keyring for security + _load_modules("keyring") profile_name = self.config.profile or "default" keyring_available = False # Nothing goes to the keyring when the profile can't be saved. @@ -753,6 +790,7 @@ def _request_device_code( Returns: DeviceCodeResponse with device code and verification URI """ + _load_modules("httpx") payload = { "client_id": CLIENT_ID, "hostname": DEVICE_NAME, @@ -796,6 +834,7 @@ def poll_device_token( and no device name is sent. Network errors are raised (``httpx.RequestError``). """ + _load_modules("httpx") base = identity_url or check_identity_host(self.config) response = self.client.post( f"{base}/token", @@ -845,6 +884,7 @@ def _poll_for_token( identity_url: str | None = None, ) -> TokenResponse: """Poll for token using device code until approved, denied or expired.""" + _load_modules("httpx") console.print("\n[blue]Waiting for authentication...[/blue]") start_time = time.time() @@ -886,6 +926,7 @@ def _poll_for_token( raise AuthenticationError("Authentication timed out") + @_forgets_resolved def _store_token(self, token_response: TokenResponse) -> None: """Store authentication token. @@ -893,6 +934,7 @@ def _store_token(self, token_response: TokenResponse) -> None: expires independently of the OIDC access token and requires a new device authorization when it expires. """ + _load_modules("keyring") profile_name = self.config.profile or "default" if token_response.dg_token: @@ -944,6 +986,7 @@ def _store_token(self, token_response: TokenResponse) -> None: "Deepgram sign-in did not return a finite credential. Run `dg update` and `dg login` again." ) + @_forgets_resolved def logout(self, keep_config: bool = False) -> None: """Logout user and clear credentials. @@ -951,6 +994,7 @@ def logout(self, keep_config: bool = False) -> None: keep_config: If True, keep profile configuration (project ID, base URL) and only clear credentials. If False, remove profile completely. """ + _load_modules("keyring") profile_name = self.config.profile or "default" # Nothing is cleared from the keyring when the profile can't be saved. self.config.check_saveable() @@ -992,9 +1036,8 @@ def list_profiles(self) -> ProfilesResult: project_id = profile.project_id try: - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) + # Bounded off a terminal; a timeout falls back like an error. + api_key = credentials.keyring_read(f"api-key.{profile_name}") # Project ID is only stored in config, not keyring except Exception: # Fall back to config @@ -1017,5 +1060,6 @@ def list_profiles(self) -> ProfilesResult: def __del__(self) -> None: """Cleanup HTTP client.""" - if hasattr(self, "client"): - self.client.close() + client = self.__dict__.get("_client") + if client is not None: + client.close() diff --git a/packages/deepctl-core/src/deepctl_core/base_command.py b/packages/deepctl-core/src/deepctl_core/base_command.py index 1b930b31..914e6dca 100644 --- a/packages/deepctl-core/src/deepctl_core/base_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_command.py @@ -6,6 +6,7 @@ import click +from . import credentials from .auth import AuthManager from .client import DeepgramClient from .config import Config, flush_config_notices @@ -153,6 +154,7 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: with TimingContext(f"command_{self.name}_total"): # Output mode is set up by now; print queued config notices. flush_config_notices() + credentials.flush_warnings() with TimingContext("command_setup"): # Get configuration from context config = ctx.obj.get("config") @@ -172,7 +174,7 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: ) # Create Deepgram client - client = DeepgramClient(config, auth_manager) + client = self._make_client(config, auth_manager) # Refusals that need no credentials come before the auth check, # so a refused command sends nothing at all. @@ -189,39 +191,12 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: with TimingContext("authentication_check"): try: auth_manager.guard() - - # Log credential source and project ID for transparency - if not config.get("output.quiet", False): - source = auth_manager.get_credential_source() - project_id = auth_manager.get_project_id() - - # Only log if not using a profile (i.e., using env vars or flags) - if source in [ - "explicit flags", - "environment variables", - ]: - # Diagnostics, not the result. print_info / - # print_warning still write to stdout outside - # agentic mode, so with an explicit `-o json` - # these three lines landed in front of the - # payload and broke `| jq` (#104, success - # path). Route them straight to stderr; the - # prefixes mirror output.py so the rendering - # is unchanged in both modes. - info = "INFO:" if _agentic else "[blue]ℹ[/blue]" - stderr_console.print( - f"{info} Using credentials from {source}" - ) - if project_id: - stderr_console.print( - f"{info} Affecting project: {project_id}" - ) - else: - warn = "WARN:" if _agentic else "[yellow]⚠[/yellow]" - stderr_console.print( - f"{warn} No project ID specified" - ) - + self._print_credential_source(config, auth_manager) + except TransportError as e: + # auth_required: the coded result on stdout and two + # stderr lines, like any other coded error. + self._report_transport_error(e, ctx, config, auth_manager) + raise SystemExit(1) from None except Exception as auth_error: # guard() already wrote the human-readable diagnosis to # stderr, so don't duplicate it -- but stdout must still @@ -310,6 +285,37 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: if exit_code: raise SystemExit(exit_code) + def _make_client(self, config: Config, auth_manager: AuthManager) -> Any: + """The Deepgram client handed to ``handle``.""" + return DeepgramClient(config, auth_manager) + + def _print_credential_source( + self, config: Config, auth_manager: AuthManager + ) -> None: + """Say where the credentials came from when no profile supplied them. + + Diagnostics, not the result: straight to stderr, so `-o json` + output stays parseable (#104). The source is the resolver's, so a + key from DEEPGRAM_API_KEY is reported as the environment's even + though Config merges it into the profile. + "No project ID specified" only for commands that need one. + """ + if config.get("output.quiet", False): + return + source = auth_manager.get_credential_source() + if source not in ("explicit flags", "environment variables"): + return + project_id = auth_manager.get_project_id() + info = "INFO:" if _agentic else "[blue]ℹ[/blue]" + stderr_console.print(f"{info} Using credentials from {source}") + if project_id: + from rich.markup import escape + + stderr_console.print(f"{info} Affecting project: {escape(project_id)}") + elif self.requires_project: + warn = "WARN:" if _agentic else "[yellow]⚠[/yellow]" + stderr_console.print(f"{warn} No project ID specified") + def _run_handler( self, config: Config, diff --git a/packages/deepctl-core/src/deepctl_core/base_group_command.py b/packages/deepctl-core/src/deepctl_core/base_group_command.py index 1c64e565..768d1981 100644 --- a/packages/deepctl-core/src/deepctl_core/base_group_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_group_command.py @@ -1,6 +1,6 @@ """Base class for group commands that can contain subcommands.""" -from typing import Any +from typing import Any, cast import click @@ -16,6 +16,34 @@ console = stderr_console +class _LazyClient: + """A :class:`DeepgramClient` built on first use. + + A group's callback runs before its subcommand parses ``--help``, so + ``dg debug --help`` must not load the SDK (and with it httpx and + ssl) just to hand the group a client it rarely uses. + """ + + __slots__ = ("_args", "_client") + + def __init__(self, config: Config, auth_manager: AuthManager) -> None: + object.__setattr__(self, "_args", (config, auth_manager)) + object.__setattr__(self, "_client", None) + + def _real(self) -> DeepgramClient: + client = object.__getattribute__(self, "_client") + if client is None: + client = DeepgramClient(*object.__getattribute__(self, "_args")) + object.__setattr__(self, "_client", client) + return cast("DeepgramClient", client) + + def __getattr__(self, name: str) -> Any: + return getattr(self._real(), name) + + def __setattr__(self, name: str, value: Any) -> None: + setattr(self._real(), name, value) + + class BaseGroupCommand(BaseCommand): """Base class for commands that act as groups containing subcommands. @@ -44,6 +72,10 @@ def __init__(self) -> None: if not hasattr(self, "invoke_without_command"): self.invoke_without_command = False + def _make_client(self, config: Config, auth_manager: AuthManager) -> Any: + """A client built only if the group's handler uses it.""" + return _LazyClient(config, auth_manager) + def add_subcommand(self, name: str, command_class: type[BaseCommand]) -> None: """Add a subcommand to this group. diff --git a/packages/deepctl-core/src/deepctl_core/credentials.py b/packages/deepctl-core/src/deepctl_core/credentials.py new file mode 100644 index 00000000..b3bd5786 --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/credentials.py @@ -0,0 +1,541 @@ +"""One place that decides which API key and project ID a run uses. + +:func:`resolve` returns a :class:`Resolved`: the key, the project ID, where +each came from, the profile and why that profile, and whether the user +config file's key was left unused because the file isn't trusted. Every +``AuthManager`` question (``get_api_key``, ``get_project_id``, +``get_credential_source``, ``is_authenticated``, ``has_profile_credentials``) +is answered from it, so ``mcp``, ``init``, ``debug toolkit`` and ``login`` +share one precedence. + +The 0.5.0 precedence is 0.4.2's, unchanged: ``--api-key`` > the profile's +keyring entry > the profile in the config file > ``DEEPGRAM_API_KEY``. The +config file layer already holds ``DEEPGRAM_API_KEY`` when it is set +(``Config`` merges the environment into the profile), so a file key equal to +the environment value is reported as the environment's. The 1.0 order +comes later. + +Under a hard AI-agent signal, ``Config`` drops the key of a user file that +others can write or that someone else owns; +``Config.api_key_untrusted`` says so and the result carries +``untrusted_source``. Such a file's ``dg_token_expires_at`` never deletes +the keyring entry. + +The keyring is imported only when a key is needed and no ``--api-key`` was +given. Off a terminal (and so under MCP), a read gets +:data:`KEYRING_BOUND_S` seconds in a daemon thread; when it doesn't answer, +the run goes on without it, says so once (``W-KEYRING-TIMEOUT``) and doesn't +ask again. + +A value that is still an unexpanded ``${...}`` placeholder counts as unset +(``W-ENV-PLACEHOLDER``, once per variable). + +This module imports only the standard library at import time. +""" + +from __future__ import annotations + +import os +import sys +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .config import Config + +# ``key_source`` values: `dg whoami`'s words, plus the flag. +KEY_SOURCE_FLAG = "--api-key" +KEY_SOURCE_KEYRING = "system keyring" +KEY_SOURCE_FILE = "config file" +KEY_SOURCE_ENV = "DEEPGRAM_API_KEY (env)" +KEY_SOURCE_NONE = "not set" + +# ``profile_source`` values (DX-S15). +PROFILE_SOURCE_NONE = "none" +PROFILE_SOURCE_FLAG = "--profile" +PROFILE_SOURCE_ENV = "DEEPGRAM_PROFILE" +PROFILE_SOURCE_FILE = "user-file default" + +# Keyring reads off a terminal get this long before the run goes on without +# the keyring. +KEYRING_BOUND_S = 2.0 + +# The env-vs-stored mismatch warning. Off in 0.5.0: login doesn't +# write fingerprints yet. A later release turns it on. +WARN_ENV_MISMATCH = False + +# The environment variables a placeholder can't stand in for. +PLACEHOLDER_VARIABLES = ( + "DEEPGRAM_API_KEY", + "DEEPGRAM_PROFILE", + "DEEPGRAM_PROJECT_ID", + "DEEPGRAM_BASE_URL", +) + +# Catalogue text (error_codes.json "strings"). Filled in with +# str.replace: the text itself holds braces. +W_ENV_PLACEHOLDER = ( + "{prog} ignored {variable}: it holds a placeholder that was never filled in (${…})." +) +W_KEYRING_TIMEOUT = ( + "The system keyring didn't answer within 2 s, so {prog} didn't use the key " + "stored there." +) + +_EXPIRY_INVALID = "Your Deepgram credential expiry is invalid. Run `dg login` again." +_EXPIRY_PAST = "Your Deepgram credential has expired. Run `dg login` again." + + +@dataclass(frozen=True) +class Resolved: + """The credentials one run uses, and where they came from.""" + + api_key: str | None + project_id: str | None + key_source: str + profile: str + profile_source: str + # The user config file whose key wasn't used (display path), or None. + untrusted_source: str | None = None + # Why a stored credential was unusable when nothing else was set. + expiry_error: str | None = None + # 0.4.2's has_profile_credentials(): a usable keyring entry, or a key in + # the profile as Config holds it (which includes DEEPGRAM_API_KEY). + profile_has_key: bool = False + profile_has_project: bool = False + project_source: str = "none" # "flag", "profile", "env" or "none" + # DEEPGRAM_API_KEY is set and differs from the stored key that won. + # Always False while WARN_ENV_MISMATCH is off. + env_mismatch: bool = False + # The keyring entry an expired credential in a trusted file leaves behind + # (``api-key.``); removed only by a resolution with prune=True. + prune_target: str | None = None + + +# ── Placeholders ───────────────────────────────────────────────────── + +_lock = threading.Lock() +# variable -> the placeholder text it held (to recognise it inside Config). +_dropped: dict[str, str] = {} +_warned: set[str] = set() + + +def is_placeholder(value: object) -> bool: + """True for a value that still holds an unexpanded ``${...}``.""" + return isinstance(value, str) and "${" in value + + +def drop_env_placeholders() -> list[str]: + """Remove placeholder values of :data:`PLACEHOLDER_VARIABLES` from the + environment, so ``Config`` and everything after it see them as unset. + + Returns the variables removed by this call. Safe to call more than once. + """ + removed = [] + with _lock: + for name in PLACEHOLDER_VARIABLES: + value = os.environ.get(name) + if is_placeholder(value): + _dropped[name] = str(value) + del os.environ[name] + removed.append(name) + return removed + + +def env_value(name: str) -> str | None: + """``os.environ[name]``, with an empty value or a placeholder treated as + unset.""" + return _env(name) + + +def _env(name: str) -> str | None: + """``os.environ[name]``, with a placeholder treated as unset.""" + value = os.environ.get(name) + if is_placeholder(value): + with _lock: + _dropped.setdefault(name, str(value)) + return None + return value or None + + +def _was_placeholder(name: str, value: object) -> bool: + """True when ``value`` is the placeholder ``name`` holds or held (merged + by a Config built before :func:`drop_env_placeholders` ran).""" + if not is_placeholder(value): + return False + if os.environ.get(name) == value: + with _lock: + _dropped.setdefault(name, str(value)) + return True + with _lock: + return _dropped.get(name) == value + + +def flush_warnings() -> None: + """Print ``W-ENV-PLACEHOLDER`` once per ignored variable.""" + with _lock: + names = [n for n in PLACEHOLDER_VARIABLES if n in _dropped and n not in _warned] + _warned.update(names) + for name in names: + _warn(W_ENV_PLACEHOLDER.replace("{variable}", name)) + + +def _warn(text: str) -> None: + from rich.markup import escape + + from . import output, transport + + message = text.replace("{prog}", transport.prog() or "dg") + output.print_warning(escape(message), stderr=True, ignore_quiet=True) + + +# ── Keyring ────────────────────────────────────────────────────────── + +_keyring_timed_out = False + + +def _interactive() -> bool: + """A person at a terminal (where a keyring may ask to be unlocked).""" + try: + return bool(sys.stdin and sys.stdin.isatty() and sys.stderr.isatty()) + except (AttributeError, ValueError, OSError): + return False + + +def keyring_timed_out() -> bool: + """True once a keyring read in this process didn't answer in time.""" + return _keyring_timed_out + + +def _keyring_call(name: str, *args: str) -> Any: + """``keyring.(*args)``, bounded off a terminal. + + Raises what the keyring raises. Returns ``None`` without calling it once + a read has timed out in this process. + """ + global _keyring_timed_out + if _keyring_timed_out: + return None + from . import auth + + def call() -> Any: + # Through deepctl_core.auth, so tests that patch auth.keyring apply; + # the import itself runs here too, inside the bound. + module = vars(auth).get("keyring") or auth._bind_module("keyring") + return getattr(module, name)(*args) + + if _interactive(): + return call() + box: dict[str, Any] = {} + + def work() -> None: + try: + box["value"] = call() + except BaseException as exc: # handed to the caller below + box["error"] = exc + + thread = threading.Thread(target=work, name="deepctl-keyring", daemon=True) + thread.start() + thread.join(KEYRING_BOUND_S) + if thread.is_alive(): + with _lock: + first = not _keyring_timed_out + _keyring_timed_out = True + if first: + _warn(W_KEYRING_TIMEOUT) + return None + if "error" in box: + raise box["error"] + return box.get("value") + + +class KeyringTimeout(Exception): + """The keyring didn't answer within :data:`KEYRING_BOUND_S` (this read or + an earlier one in this process).""" + + +def keyring_read(username: str) -> str | None: + """The keyring's value for ``username``, bounded like every resolver read. + + Raises what the keyring raises, and :class:`KeyringTimeout` when it + doesn't answer in time, so callers that fall back on an error keep doing + so. Returns ``None`` when there's no entry. + """ + from .auth import KEYRING_SERVICE + + if _keyring_timed_out: + raise KeyringTimeout() + value = _keyring_call("get_password", KEYRING_SERVICE, username) + if _keyring_timed_out: + raise KeyringTimeout() + return value if isinstance(value, str) and value else None + + +def keyring_get(username: str) -> str | None: + """The keyring's value for ``username``; ``None`` if absent, failing or + timed out.""" + try: + return keyring_read(username) + except Exception: + return None + + +def keyring_delete(username: str) -> None: + """Remove a keyring entry when present, without masking keyring failures.""" + from .auth import KEYRING_SERVICE, console + + try: + _keyring_call("delete_password", KEYRING_SERVICE, username) + except Exception as exc: + from keyring.errors import PasswordDeleteError + + if isinstance(exc, PasswordDeleteError): + return + console.print( + f"[yellow]Warning:[/yellow] Could not remove legacy credential: {exc}" + ) + + +# ── Resolution ─────────────────────────────────────────────────────── + + +def credential_expiry(config: Config, profile_name: str) -> tuple[bool, str | None]: + """Whether the profile's finite credential is unusable, and why.""" + expires_at = config.get_profile(profile_name).dg_token_expires_at + if not isinstance(expires_at, str): + return False, None + try: + parsed = datetime.fromisoformat(expires_at) + if parsed.tzinfo is None: + raise ValueError + except (TypeError, ValueError): + return True, _EXPIRY_INVALID + if parsed <= datetime.now(timezone.utc): + return True, _EXPIRY_PAST + return False, None + + +def _user_value(config: Config, *path: str) -> Any: + data: Any = getattr(config, "_user_data", None) + for part in path: + if not isinstance(data, dict): + return None + data = data.get(part) + return data + + +def profile_name(config: Config) -> tuple[str, str]: + """The profile this run uses and why (``profile_source``).""" + name = config.profile or "default" + explicit = getattr(config, "_explicit_profile", None) + env_profile = os.environ.get("DEEPGRAM_PROFILE") + if explicit and name == explicit: + return name, PROFILE_SOURCE_FLAG + if is_placeholder(name) and _was_placeholder("DEEPGRAM_PROFILE", name): + # Config took the placeholder as the profile name: use the one the + # file names instead. + name = ( + _user_value(config, "active_profile") + or _user_value(config, "default_profile") + or "default" + ) + name = str(name) + source = PROFILE_SOURCE_FILE if name != "default" else PROFILE_SOURCE_NONE + return name, source + if config.get("active_profile") == name: + return name, PROFILE_SOURCE_FILE + if env_profile and name == env_profile: + return name, PROFILE_SOURCE_ENV + if _user_value(config, "default_profile") == name: + return name, PROFILE_SOURCE_FILE + return name, PROFILE_SOURCE_NONE + + +def _profile_key(config: Config, name: str) -> str | None: + """The profile's key as Config holds it (file, or the merged env value).""" + key = config.get_profile(name).api_key + if not key: + return None + if is_placeholder(key): + if not _was_placeholder("DEEPGRAM_API_KEY", key): + return None + # Config merged the placeholder over the file's key: use the file's. + if getattr(config, "api_key_untrusted", False) is True: + return None + stored = _user_value(config, "profiles", name, "api_key") + return ( + stored if isinstance(stored, str) and not is_placeholder(stored) else None + ) + return key + + +def _profile_project(config: Config, name: str) -> str | None: + project = config.get_profile(name).project_id + if not project: + return None + if is_placeholder(project): + if not _was_placeholder("DEEPGRAM_PROJECT_ID", project): + return None + stored = _user_value(config, "profiles", name, "project_id") + return ( + stored if isinstance(stored, str) and not is_placeholder(stored) else None + ) + return project + + +def resolve( + ctx: Any, + *, + profile: str | None = None, + ignore_env: bool = False, + need_key: bool = True, + prune: bool = True, +) -> Resolved: + """The credentials for ``ctx``: anything with ``config``, + ``explicit_api_key`` and ``explicit_project_id`` (an ``AuthManager``). + + ``profile`` asks about another profile than the run's. ``ignore_env`` + leaves ``DEEPGRAM_API_KEY`` and ``DEEPGRAM_PROJECT_ID`` out. + ``need_key=False`` answers project questions without reading the keyring. + ``prune=False`` keeps the answer read-only: an expired credential's + keyring entry stays (whoami and the other introspection, as in 0.4.2); + commands that use the key prune it, as 0.4.2's ``get_api_key`` did. + Raises nothing; ``expiry_error`` says why a stored credential wasn't used. + """ + try: + resolved = _resolve(ctx, profile, ignore_env, need_key) + if prune: + prune_expired(resolved) + return resolved + finally: + flush_warnings() + + +def prune_expired(resolved: Resolved) -> None: + """Remove the keyring entry of an expired credential (trusted file only).""" + if resolved.prune_target: + keyring_delete(resolved.prune_target) + + +def _resolve( + ctx: Any, profile: str | None, ignore_env: bool, need_key: bool +) -> Resolved: + config: Config = ctx.config + if profile: + name, profile_source = profile, PROFILE_SOURCE_FLAG + else: + name, profile_source = profile_name(config) + untrusted = getattr(config, "api_key_untrusted", False) is True + untrusted_source = None + if untrusted: + from . import messages + + untrusted_source = messages.display_path(config.config_path) + + env_key = None if ignore_env else _env("DEEPGRAM_API_KEY") + env_project = None if ignore_env else _env("DEEPGRAM_PROJECT_ID") + + # Project ID: flag > profile (as Config holds it) > env. + project_file = _profile_project(config, name) + raw_env_project = _env("DEEPGRAM_PROJECT_ID") + if ctx.explicit_project_id: + project_id, project_source = ctx.explicit_project_id, "flag" + elif project_file: + project_id = project_file + project_source = ( + "env" if raw_env_project and project_file == raw_env_project else "profile" + ) + elif env_project: + project_id, project_source = env_project, "env" + else: + project_id, project_source = None, "none" + + def result( + key: str | None, + source: str, + *, + has_key: bool = False, + expiry_error: str | None = None, + env_mismatch: bool = False, + ) -> Resolved: + # Only a trusted file's expiry may remove the keyring entry. + prune_target = f"api-key.{name}" if expired and not untrusted else None + return Resolved( + api_key=key, + project_id=project_id, + key_source=source, + profile=name, + profile_source=profile_source, + untrusted_source=untrusted_source, + expiry_error=expiry_error, + profile_has_key=has_key, + profile_has_project=bool(project_file), + project_source=project_source, + env_mismatch=env_mismatch, + prune_target=prune_target, + ) + + expired = False + if ctx.explicit_api_key: + return result(ctx.explicit_api_key, KEY_SOURCE_FLAG) + if not need_key: + return result(None, KEY_SOURCE_NONE) + + file_key = _profile_key(config, name) + raw_env_key = _env("DEEPGRAM_API_KEY") + expired, expiry_error = credential_expiry(config, name) + if not expired: + stored = keyring_get(f"api-key.{name}") + if stored: + mismatch = _mismatch(stored, env_key) + return result( + stored, KEY_SOURCE_KEYRING, has_key=True, env_mismatch=mismatch + ) + if file_key: + source = ( + KEY_SOURCE_ENV + if raw_env_key and file_key == raw_env_key + else KEY_SOURCE_FILE + ) + # As in 0.4.2, ignore_env doesn't skip the env value Config + # merged into the profile; it skips only the last fallback. + mismatch = source == KEY_SOURCE_FILE and _mismatch(file_key, env_key) + return result(file_key, source, has_key=True, env_mismatch=mismatch) + + if env_key: + return result(env_key, KEY_SOURCE_ENV, expiry_error=expiry_error) + return result(None, KEY_SOURCE_NONE, expiry_error=expiry_error) + + +# ── Env-vs-stored mismatch ─────────────────────────────────────────── + + +def fingerprint(key: str) -> str: + """A short, non-reversible fingerprint of a key, for comparisons.""" + import hashlib + + return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + + +def _mismatch(stored: str, env_key: str | None) -> bool: + """True when DEEPGRAM_API_KEY is set and isn't the stored key that won. + + Compares fingerprints, never keys, so a later release can compare with the + fingerprint login will store. Behind :data:`WARN_ENV_MISMATCH`, which is + off in 0.5.0: login doesn't write fingerprints yet, and a later release + adds the warning text and turns it on. + """ + if not WARN_ENV_MISMATCH or not env_key: + return False + return fingerprint(stored) != fingerprint(env_key) + + +def _reset_for_tests() -> None: + """Forget the per-process state (placeholders, warnings, keyring timeout).""" + global _keyring_timed_out + with _lock: + _dropped.clear() + _warned.clear() + _keyring_timed_out = False diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py index bb1030c5..d4ec6a16 100644 --- a/packages/deepctl-core/src/deepctl_core/transport.py +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -28,6 +28,8 @@ import threading from typing import TYPE_CHECKING, Any +from . import credentials + if TYPE_CHECKING: import httpx @@ -95,6 +97,25 @@ def _set_client(client: httpx.Client | None) -> None: # ── Catalogue text (error_codes.json "strings") ────────────────────── STRINGS: dict[str, str] = { + # The 0.4.x error text, frozen: scripts match on it (legacy_error_text_frozen). + "E-AUTH-REQUIRED": "DEEPGRAM_API_KEY is not set", + "H-AUTH-REQUIRED": "Run {prog} login, or set DEEPGRAM_API_KEY.", + "A-AUTH-REQUIRED": ( + "Run {prog} login and give the user the verification link and code it " + "prints. Never ask the user for an API key in a chat, and don't search " + "files for one." + ), + "C-AUTH-REQUIRED": ( + "Set DEEPGRAM_API_KEY in this job's environment, or run {prog} login." + ), + # 0.4.1's messages.E_KEY_UNTRUSTED and W_CFG_UNTRUSTED_FILE_HINT, verbatim. + "E-KEY-UNTRUSTED": ( + "dg didn't use the API key in {user_config} (see the warning above), and no " + "other API key is set." + ), + "W_CFG_UNTRUSTED_FILE_HINT": ( + "A person must fix this file. If you are an AI agent, ask the user." + ), "E-AUTH-INVALID": "Deepgram rejected the API key (HTTP 401).", "H-AUTH-INVALID": ( "The API key may have expired or been revoked. Run {prog} login to log " @@ -193,6 +214,12 @@ def _set_client(client: httpx.Client | None) -> None: # code -> (error message ID, human hint, agent hint, automation hint or None) Messages = tuple[str, str, str, "str | None"] CODES: dict[str, Messages] = { + "auth_required": ( + "E-AUTH-REQUIRED", + "H-AUTH-REQUIRED", + "A-AUTH-REQUIRED", + "C-AUTH-REQUIRED", + ), "auth_invalid": ( "E-AUTH-INVALID", "H-AUTH-INVALID", @@ -228,6 +255,16 @@ def _set_client(client: httpx.Client | None) -> None: None, ) +# auth_required's other message: the only key was in a user config file this +# AI agent session doesn't trust. next_step is null: a person +# has to fix the file. +KEY_UNTRUSTED: Messages = ( + "E-KEY-UNTRUSTED", + "W_CFG_UNTRUSTED_FILE_HINT", + "W_CFG_UNTRUSTED_FILE_HINT", + None, +) + # Network failure reasons for E-NETWORK's {reason}. REASON_DNS = "name resolution failed" REASON_REFUSED = "connection refused" @@ -239,10 +276,10 @@ def _set_client(client: httpx.Client | None) -> None: REASON_PROXY = "the proxy refused the connection" REASON_OTHER = "connection failed" -KEY_SOURCE_ENV = "DEEPGRAM_API_KEY (env)" -KEY_SOURCE_FLAG = "--api-key" -KEY_SOURCE_KEYRING = "system keyring" -KEY_SOURCE_FILE = "config file" +KEY_SOURCE_ENV = credentials.KEY_SOURCE_ENV +KEY_SOURCE_FLAG = credentials.KEY_SOURCE_FLAG +KEY_SOURCE_KEYRING = credentials.KEY_SOURCE_KEYRING +KEY_SOURCE_FILE = credentials.KEY_SOURCE_FILE class TransportError(BaseException): @@ -517,33 +554,17 @@ def prog() -> str | None: def _key_source(auth_manager: AuthManager | None) -> str | None: + """Where the key that was sent came from: the resolver's answer, so the + error path reads the keyring no second time.""" if auth_manager is None: return None if auth_manager.explicit_api_key: return KEY_SOURCE_FLAG try: - key = auth_manager.get_api_key() + source = auth_manager.resolve().key_source except Exception: return None - if not key: - return None - # The order `dg whoami` uses: keyring, then the config file (a profile key - # equal to the env value came from the env), then the env. - import keyring - - from .auth import KEYRING_SERVICE - - env_key = os.environ.get("DEEPGRAM_API_KEY") - profile = auth_manager.config.profile or "default" - try: - stored = keyring.get_password(KEYRING_SERVICE, f"api-key.{profile}") - except Exception: - stored = None - if stored and stored == key: - return KEY_SOURCE_KEYRING - # Config merges the env key into the profile, so a file key equal to the - # env value came from the env. - return KEY_SOURCE_ENV if env_key == key else KEY_SOURCE_FILE + return None if source == credentials.KEY_SOURCE_NONE else source def render( @@ -564,7 +585,12 @@ def render( next_step: str | None = None hint_id = human if not agentic else (automation or agent) if soft_only else agent - if err.code == "auth_invalid": + if err.code == "auth_required": + # Nobody can approve a device code under soft-only signals (CI, cron), + # and a person has to fix an untrusted file. + if not soft_only and program and err.messages[0] == "E-AUTH-REQUIRED": + next_step = f"{program} login" + elif err.code == "auth_invalid": source = _key_source(auth_manager) if source: details["key_source"] = source diff --git a/packages/deepctl-core/tests/unit/test_auth.py b/packages/deepctl-core/tests/unit/test_auth.py index 946f1e11..229e5823 100644 --- a/packages/deepctl-core/tests/unit/test_auth.py +++ b/packages/deepctl-core/tests/unit/test_auth.py @@ -602,7 +602,10 @@ class TestFiniteCredentialFlow: def _make_auth(self, mock_config): with patch("deepctl_core.auth.httpx.Client"): with patch("deepctl_core.auth.keyring"): - return AuthManager(mock_config) + auth = AuthManager(mock_config) + # The client is built on first use; build it under the patch. + _ = auth.client + return auth # ------------------------------------------------------------------ # _store_token — finite direct credential path diff --git a/packages/deepctl-core/tests/unit/test_auth_insecure.py b/packages/deepctl-core/tests/unit/test_auth_insecure.py index 88bfd1df..a1e3f810 100644 --- a/packages/deepctl-core/tests/unit/test_auth_insecure.py +++ b/packages/deepctl-core/tests/unit/test_auth_insecure.py @@ -25,7 +25,8 @@ def _isolated(monkeypatch): def client_kwargs(config) -> dict: with patch.object(auth_module.httpx, "Client") as client: - AuthManager(config) + # The client is built on first use (login's device flow and verify). + _ = AuthManager(config).client return dict(client.call_args.kwargs) diff --git a/packages/deepctl-core/tests/unit/test_config_location.py b/packages/deepctl-core/tests/unit/test_config_location.py index a7dfbb9a..9b010c1a 100644 --- a/packages/deepctl-core/tests/unit/test_config_location.py +++ b/packages/deepctl-core/tests/unit/test_config_location.py @@ -787,10 +787,21 @@ def test_missing_key_after_untrusted_file_key_says_so(real_home, monkeypatch, ca "above), and no other API key is set." ) assert str(err.value) == expected - captured = capsys.readouterr() - text = (captured.out + captured.err).replace("\n", " ") - assert f"ERROR: {expected}" in text - assert f"HINT: {FILE_HINT}" in text + # guard() raises the coded auth_required (E-KEY-UNTRUSTED); + # BaseCommand prints its two lines (test_credentials.py covers that). + from deepctl_core import transport + + assert isinstance(err.value, transport.TransportError) + assert err.value.code == "auth_required" + output._output_config.update(agentic=True) + try: + result = transport.render(err.value, command_path="listen") + finally: + output._output_config.update(agentic=False) + assert result.error == expected + assert result.hint == FILE_HINT + assert result.next_step is None + text = f"{result.error} {result.hint}" assert "not set in the configuration file" not in text assert "deepctl login" not in text assert "dg login" not in text @@ -921,6 +932,16 @@ def deny(*args, **kwargs): assert len(texts) == 1 and "set DEEPGRAM_API_KEY instead" in texts[0] +def _render_guard_error(manager, error) -> None: + """The two stderr lines BaseCommand prints for guard()'s coded error.""" + from deepctl_core import transport + from rich.markup import escape + + result = transport.render(error, command_path="projects", auth_manager=manager) + output.print_error(escape(result.error)) + output.print_hint(escape(result.hint)) + + def _run_projects_guard(capsys): """Config load + notices + guard, as a `dg projects` run renders them.""" from deepctl_core.auth import AuthenticationError, AuthManager @@ -929,8 +950,10 @@ def _run_projects_guard(capsys): try: config = Config() config_module.flush_config_notices() - with pytest.raises(AuthenticationError): - AuthManager(config).guard() + manager = AuthManager(config) + with pytest.raises(AuthenticationError) as raised: + manager.guard() + _render_guard_error(manager, raised.value) finally: output._output_config.update(agentic=False) return capsys.readouterr().err @@ -973,8 +996,10 @@ def test_untrusted_hint_follows_the_error_when_nothing_warned(real_home, monkeyp plant_with_key(real_home, 0o666) output._output_config.update(agentic=True) try: - with pytest.raises(AuthenticationError): - AuthManager(Config()).guard() + manager = AuthManager(Config()) + with pytest.raises(AuthenticationError) as raised: + manager.guard() + _render_guard_error(manager, raised.value) finally: output._output_config.update(agentic=False) lines = [line for line in capsys.readouterr().err.splitlines() if line.strip()] diff --git a/packages/deepctl-core/tests/unit/test_credentials.py b/packages/deepctl-core/tests/unit/test_credentials.py new file mode 100644 index 00000000..3cf3a01e --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_credentials.py @@ -0,0 +1,630 @@ +"""deepctl_core.credentials: one resolver, 0.4.2's precedence.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +import yaml +from deepctl_core import auth as auth_module +from deepctl_core import config as config_module +from deepctl_core import credentials, hosts, output, transport +from deepctl_core.auth import AuthManager, AuthRequiredError +from deepctl_core.config import Config + +pytestmark = pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="XDG layout (Linux)" +) + +KEYRING_SERVICE = auth_module.KEYRING_SERVICE + + +class FakeKeyring: + """A dict-backed keyring; ``hang`` makes reads block until released.""" + + def __init__(self, entries: dict[str, str] | None = None) -> None: + self.entries = dict(entries or {}) + self.reads: list[str] = [] + self.deletes: list[str] = [] + self.hang: threading.Event | None = None + + def get_password(self, service: str, username: str) -> str | None: + assert service == KEYRING_SERVICE + self.reads.append(username) + if self.hang is not None: + self.hang.wait(10) + return self.entries.get(username) + + def delete_password(self, service: str, username: str) -> None: + self.deletes.append(username) + self.entries.pop(username, None) + + def set_password(self, service: str, username: str, value: str) -> None: + self.entries[username] = value + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + monkeypatch.setattr(hosts, "_reported", set()) + home = tmp_path / "home" + home.mkdir() + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + for name in output.AI_AGENT_ENV_MARKERS: + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setattr(transport, "prog", lambda: "dg") + return home + + +@pytest.fixture +def kr(monkeypatch) -> FakeKeyring: + fake = FakeKeyring() + monkeypatch.setattr(auth_module, "keyring", fake, raising=False) + return fake + + +def write_config(home: Path, data: dict[str, Any], mode: int = 0o600) -> Path: + directory = home / ".config" / "deepctl" + directory.mkdir(parents=True, exist_ok=True) + path = directory / "config.yaml" + path.write_text(yaml.safe_dump(data)) + os.chmod(path, mode) + return path + + +# ── 0.4.2's get_api_key(), verbatim in substance, as the golden ───────── + + +def golden_042(config: Config, explicit: str | None, kr: FakeKeyring) -> str | None: + if explicit: + return explicit + profile_name = config.profile or "default" + current_profile = config.get_profile(profile_name) + expired, _ = credentials.credential_expiry(config, profile_name) + if not expired: + api_key = kr.entries.get(f"api-key.{profile_name}") + if api_key: + return api_key + if current_profile.api_key: + return current_profile.api_key + return os.getenv("DEEPGRAM_API_KEY") or None + + +def whoami_source_042(config: Config, key: str | None, kr: FakeKeyring) -> str: + """`dg whoami`'s key_source rule (0.4.2), plus the flag.""" + if key is None: + return credentials.KEY_SOURCE_NONE + name = config.profile or "default" + if kr.entries.get(f"api-key.{name}") == key: + return credentials.KEY_SOURCE_KEYRING + env = os.getenv("DEEPGRAM_API_KEY") + if config.get_profile(name).api_key == key: + return credentials.KEY_SOURCE_ENV if env == key else credentials.KEY_SOURCE_FILE + return credentials.KEY_SOURCE_ENV + + +PROFILE_SOURCES = ("none", "--profile", "DEEPGRAM_PROFILE", "user-file default") + + +@pytest.mark.parametrize("profile_source", PROFILE_SOURCES) +@pytest.mark.parametrize("flag", [False, True]) +@pytest.mark.parametrize("keyring_set", [False, True]) +@pytest.mark.parametrize("file_set", [False, True]) +@pytest.mark.parametrize("env_set", [False, True]) +def test_precedence_matches_042( + home, kr, monkeypatch, profile_source, flag, keyring_set, file_set, env_set +): + profile = "default" if profile_source == "none" else "work" + data: dict[str, Any] = {"profiles": {}} + if file_set: + data["profiles"][profile] = {"api_key": "f" * 39 + "1"} + if profile_source == "user-file default": + data["default_profile"] = "work" + write_config(home, data) + if keyring_set: + kr.entries[f"api-key.{profile}"] = "f" * 39 + "2" + if env_set: + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 39 + "3") + if profile_source == "DEEPGRAM_PROFILE": + monkeypatch.setenv("DEEPGRAM_PROFILE", "work") + config = Config(profile="work" if profile_source == "--profile" else None) + explicit = "f" * 39 + "4" if flag else None + + resolved = AuthManager(config, explicit).resolve() + + want = golden_042(config, explicit, kr) + assert resolved.api_key == want + assert resolved.profile == profile + assert resolved.profile_source == profile_source + if flag: + assert resolved.key_source == credentials.KEY_SOURCE_FLAG + assert kr.reads == [] # --api-key never reads the keyring + else: + assert resolved.key_source == whoami_source_042(config, want, kr) + + +def test_keyring_read_once_per_run(home, kr, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + auth = AuthManager(Config()) + auth.guard() + auth.get_api_key() + auth.get_credential_source() + auth.has_profile_credentials() + auth.is_authenticated() + auth.get_project_id() + assert transport._key_source(auth) == credentials.KEY_SOURCE_ENV + assert kr.reads == ["api-key.default"] + + +def test_login_forgets_the_cached_answer(home, kr, monkeypatch): + auth = AuthManager(Config()) + assert auth.get_api_key() is None + monkeypatch.setattr(auth, "verify_credentials", lambda *a: (True, "ok", None)) + auth.login_with_api_key("f" * 40) + assert auth.get_api_key() == "f" * 40 + + +def test_project_id_precedence(home, kr, monkeypatch): + write_config(home, {"profiles": {"default": {"project_id": "p-file"}}}) + monkeypatch.setenv("DEEPGRAM_PROJECT_ID", "p-env") + # Config merges the env value over the file's, as in 0.4.2. + assert AuthManager(Config()).get_project_id() == "p-env" + assert AuthManager(Config(), None, "p-flag").get_project_id() == "p-flag" + monkeypatch.delenv("DEEPGRAM_PROJECT_ID") + assert AuthManager(Config()).get_project_id() == "p-file" + assert kr.reads == [] # project questions don't read the keyring + + +# ── Where the credentials line says they came from ───────────────────── + + +def test_env_key_is_reported_as_the_environments(home, kr, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + auth = AuthManager(Config()) + # 0.4.2 said "profile 'default'": Config merges the env key into the profile. + assert auth.get_credential_source() == "environment variables" + assert auth.has_profile_credentials() == (True, False) # 0.4.2's answer + + +def test_stored_key_is_reported_as_the_profiles(home, kr, monkeypatch): + kr.entries["api-key.default"] = "f" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert AuthManager(Config()).get_credential_source() == "profile 'default'" + write_config(home, {"profiles": {"default": {"api_key": "d" * 40}}}) + kr.entries.clear() + monkeypatch.delenv("DEEPGRAM_API_KEY") + assert AuthManager(Config()).get_credential_source() == "profile 'default'" + assert AuthManager(Config(), "c" * 40).get_credential_source() == "explicit flags" + + +# ── Trust cells and key expiry ──────────────────────────────── + + +@pytest.mark.parametrize("mode", [0o644, 0o666, 0o660]) +@pytest.mark.parametrize("agent", [False, True]) +def test_trust_cells(home, kr, monkeypatch, mode, agent): + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, mode) + resolved = AuthManager(Config()).resolve() + writable = bool(mode & 0o022) + if agent and writable: + assert resolved.api_key is None + assert resolved.untrusted_source == "~/.config/deepctl/config.yaml" + with pytest.raises(AuthRequiredError) as err: + AuthManager(Config()).guard() + assert err.value.messages == transport.KEY_UNTRUSTED + else: + assert resolved.api_key == "f" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_FILE + assert resolved.untrusted_source is None + + +def test_untrusted_file_with_env_key_uses_env(home, kr, monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, 0o666) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + resolved = AuthManager(Config()).resolve() + assert resolved.api_key == "e" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_ENV + + +def test_sf1_untrusted_expiry_never_deletes_keyring(home, kr, monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config( + home, + {"profiles": {"default": {"api_key": "f" * 40, "dg_token_expires_at": past}}}, + 0o666, + ) + kr.entries["api-key.default"] = "k" * 40 + AuthManager(Config()).resolve() + assert kr.deletes == [] + assert kr.entries["api-key.default"] == "k" * 40 + + +def test_trusted_expiry_still_deletes_keyring(home, kr): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config(home, {"profiles": {"default": {"dg_token_expires_at": past}}}) + kr.entries["api-key.default"] = "k" * 40 + with pytest.raises(auth_module.AuthenticationError, match="expired"): + AuthManager(Config()).get_api_key() + assert kr.deletes == ["api-key.default"] + + +def test_introspection_never_prunes(home, kr, monkeypatch): + """Whoami and login's checks stay read-only, as in 0.4.2.""" + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config(home, {"profiles": {"default": {"dg_token_expires_at": past}}}) + kr.entries["api-key.default"] = "k" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + auth = AuthManager(Config()) + assert auth.has_profile_credentials() == (False, False) + assert auth.has_profile_credentials("default") == (False, False) + assert auth.is_authenticated() is True + assert kr.deletes == [] + # A command that uses the key prunes it, once (0.4.2's get_api_key). + assert auth.get_api_key() == "e" * 40 + assert auth.get_api_key() == "e" * 40 + assert kr.deletes == ["api-key.default"] + + +def test_one_resolution_per_profile_spelling(home, kr): + kr.entries["api-key.default"] = "k" * 40 + auth = AuthManager(Config()) + auth.resolve(profile="default", use_flags=False, prune=False) + auth.resolve() + auth.has_profile_credentials() + assert kr.reads == ["api-key.default"] + + +# ── Keyring bound ─────────────────────────────────────────────────────── + + +def test_keyring_bound_falls_back_once(home, kr, monkeypatch, capsys): + # A 2 s bound: the first read waits it once, a second wait would show. + monkeypatch.setattr(credentials, "KEYRING_BOUND_S", 2.0) + monkeypatch.setattr(credentials, "_interactive", lambda: False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + kr.entries["api-key.default"] = "k" * 40 + kr.hang = threading.Event() + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + try: + started = time.monotonic() + assert AuthManager(Config()).get_api_key() == "e" * 40 + assert time.monotonic() - started < 6.0 + # Cached for the rest of the process: no second wait, no second line. + started = time.monotonic() + assert AuthManager(Config()).get_api_key() == "e" * 40 + assert time.monotonic() - started < 1.0 + finally: + kr.hang.set() + err = capsys.readouterr().err + assert err.count("The system keyring didn't answer within 2 s") == 1 + assert "k" * 40 not in err and "e" * 40 not in err + assert len(kr.reads) == 1 + + +def test_keyring_unbounded_at_a_terminal(home, kr, monkeypatch): + monkeypatch.setattr(credentials, "_interactive", lambda: True) + + def no_threads(*args: Any, **kwargs: Any) -> None: + raise AssertionError("an interactive read runs in the caller's thread") + + monkeypatch.setattr(credentials.threading, "Thread", no_threads) + kr.entries["api-key.default"] = "k" * 40 + assert AuthManager(Config()).get_api_key() == "k" * 40 + + +def test_keyring_errors_fall_through(home, kr, monkeypatch): + def broken(*args: Any) -> None: + raise RuntimeError("no backend") + + monkeypatch.setattr(kr, "get_password", broken) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert AuthManager(Config()).get_api_key() == "e" * 40 + + +# ── Placeholders ──────────────────────────────────────────────────────── + + +def test_placeholder_key_counts_as_unset(home, kr, monkeypatch, capsys): + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setenv("DEEPGRAM_API_KEY", "${DEEPGRAM_API_KEY}") + assert credentials.drop_env_placeholders() == ["DEEPGRAM_API_KEY"] + auth = AuthManager(Config()) + with pytest.raises(AuthRequiredError): + auth.guard() + credentials.flush_warnings() + err = capsys.readouterr().err + assert err.count("ignored DEEPGRAM_API_KEY: it holds a placeholder") == 1 + assert "${DEEPGRAM_API_KEY}" not in err + + +def test_placeholder_merged_before_the_drop_keeps_the_file_key(home, kr, monkeypatch): + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}) + monkeypatch.setenv("DEEPGRAM_API_KEY", "${DEEPGRAM_API_KEY}") + config = Config() # e.g. `dg mcp`'s handoff builds its own Config first + resolved = AuthManager(config).resolve() + assert resolved.api_key == "f" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_FILE + + +@pytest.mark.parametrize( + "name", ["DEEPGRAM_PROFILE", "DEEPGRAM_PROJECT_ID", "DEEPGRAM_BASE_URL"] +) +def test_other_placeholders_count_as_unset(home, kr, monkeypatch, capsys, name): + monkeypatch.setattr(output.stderr_console, "_width", 400) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}) + monkeypatch.setenv(name, "${" + name + "}") + credentials.drop_env_placeholders() + config = Config() + resolved = AuthManager(config).resolve() + assert resolved.api_key == "f" * 40 + assert resolved.profile == "default" + assert resolved.project_id is None + assert hosts.check_credential_host(config) == hosts.DEFAULT_BASE_URL + credentials.flush_warnings() + credentials.flush_warnings() + err = capsys.readouterr().err + assert err.count(f"ignored {name}: it holds a placeholder") == 1 + + +def test_profile_placeholder_merged_before_the_drop(home, kr, monkeypatch): + write_config( + home, + {"default_profile": "work", "profiles": {"work": {"api_key": "f" * 40}}}, + ) + monkeypatch.setenv("DEEPGRAM_PROFILE", "${DEEPGRAM_PROFILE}") + resolved = AuthManager(Config()).resolve() + assert resolved.profile == "work" + assert resolved.api_key == "f" * 40 + + +# ── Env-vs-stored mismatch: implemented, off in 0.5.0 ────────────────── + + +def test_mismatch_check_is_off(home, kr, monkeypatch): + kr.entries["api-key.default"] = "k" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert credentials.WARN_ENV_MISMATCH is False + assert AuthManager(Config()).resolve().env_mismatch is False + monkeypatch.setattr(credentials, "WARN_ENV_MISMATCH", True) + assert AuthManager(Config()).resolve().env_mismatch is True + assert credentials.fingerprint("k" * 40) != "k" * 40 + + +# ── auth_required rendering ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("agentic", "hard", "hint_id", "next_step"), + [ + (False, False, "H-AUTH-REQUIRED", "dg login"), + (True, True, "A-AUTH-REQUIRED", "dg login"), + (True, False, "C-AUTH-REQUIRED", None), + ], +) +def test_auth_required_render(monkeypatch, agentic, hard, hint_id, next_step): + monkeypatch.setattr(output, "is_ai_agent", lambda: hard) + monkeypatch.setitem(output._output_config, "agentic", agentic) + result = transport.render(AuthRequiredError(), command_path="listen") + assert result.code == "auth_required" + assert result.error == "DEEPGRAM_API_KEY is not set" + assert result.hint == transport.STRINGS[hint_id].format(prog="dg") + assert result.next_step == next_step + assert result.model_dump()["schema_version"] == 1 + assert "details" not in result.model_dump() + + +def test_key_untrusted_render_has_no_next_step(monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + err = AuthRequiredError(untrusted_source="~/.config/deepctl/config.yaml") + result = transport.render(err, command_path="listen") + assert result.error == ( + "dg didn't use the API key in ~/.config/deepctl/config.yaml (see the " + "warning above), and no other API key is set." + ) + assert result.hint == ( + "A person must fix this file. If you are an AI agent, ask the user." + ) + assert result.next_step is None + + +def test_catalogue_text_is_0_4_1s(monkeypatch): + from deepctl_core import messages + + assert transport.STRINGS["E-KEY-UNTRUSTED"] == messages.E_KEY_UNTRUSTED + assert ( + transport.STRINGS["W_CFG_UNTRUSTED_FILE_HINT"] + == messages.W_CFG_UNTRUSTED_FILE_HINT + ) + assert isinstance(AuthRequiredError(), auth_module.AuthenticationError) + + +def _run_command(monkeypatch, capsys, *, requires_project: bool = False) -> Any: + import click + from deepctl_core.base_command import BaseCommand + from deepctl_core.models import BaseResult + + class NeedsAuth(BaseCommand): + name = "listen" + help = "test command" + requires_auth = True + + def handle(self, config, auth_manager, client, **kwargs): # type: ignore[no-untyped-def] + return BaseResult(status="success", message="done") + + NeedsAuth.requires_project = requires_project + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setattr("deepctl_core.output.get_output_format", lambda: "json") + ctx = click.Context( + click.Command("listen"), info_name="dg listen", obj={"config": Config()} + ) + code = 0 + try: + NeedsAuth().execute(ctx) + except SystemExit as exc: + code = int(exc.code or 0) + except click.ClickException: + code = 1 + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def test_no_key_prints_coded_result_and_two_lines(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 1 + payload = json.loads(out) + assert payload == { + "error": "DEEPGRAM_API_KEY is not set", + "status": "error", + "code": "auth_required", + "hint": transport.STRINGS["A-AUTH-REQUIRED"].format(prog="dg"), + "next_step": "dg login", + "schema_version": 1, + } + assert err.splitlines() == [ + "ERROR: DEEPGRAM_API_KEY is not set", + "HINT: " + transport.STRINGS["A-AUTH-REQUIRED"].format(prog="dg"), + ] + + +def test_untrusted_file_prints_two_lines(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, 0o666) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 1 + payload = json.loads(out) + assert payload["code"] == "auth_required" + assert payload["next_step"] is None + lines = err.splitlines() + # The warning's hint prints once per run (0.4.1), before the error. + assert lines[-1].startswith("ERROR: dg didn't use the API key in ") + assert lines[-2] == ( + "HINT: A person must fix this file. If you are an AI agent, ask the user." + ) + assert err.count("HINT: ") == 1 + assert "f" * 40 not in out + err + + +def test_env_key_info_line(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 0 + assert err.splitlines() == ["INFO: Using credentials from environment variables"] + + +def test_no_project_warning_only_when_needed(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + _, _, err = _run_command(monkeypatch, capsys, requires_project=True) + assert "WARN: No project ID specified" in err + + +# ── Import cost: no keyring with --api-key, no client in a group's --help ─ + + +def _imports(code: str, home: Path) -> set[str]: + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("DEEPGRAM_") and k not in output.AI_AGENT_ENV_MARKERS + } + env.update(HOME=str(home), DEEPCTL_TELEMETRY_DISABLED="1") + done = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + text=True, + env=env, + timeout=60, + ) + assert done.returncode == 0, done.stderr + return set(json.loads(done.stdout.strip().splitlines()[-1])) + + +def test_api_key_flag_never_imports_keyring(home): + modules = _imports( + """ + import json, sys + from deepctl_core.auth import AuthManager + from deepctl_core.config import Config + auth = AuthManager(Config(), "f" * 40) + auth.guard() + assert auth.get_api_key() == "f" * 40 + auth.get_credential_source() + print(json.dumps(sorted(sys.modules))) + """, + home, + ) + assert "keyring" not in modules + assert "httpx" not in modules + + +def test_group_subcommand_help_builds_no_client(home): + modules = _imports( + """ + import json, sys + import click + from deepctl_core.base_command import BaseCommand + from deepctl_core.base_group_command import BaseGroupCommand + + class Sub(BaseCommand): + name = "sub" + help = "sub" + def handle(self, config, auth_manager, client, **kwargs): + return None + + class Group(BaseGroupCommand): + name = "grp" + help = "grp" + def setup_commands(self): + return [click.Command("sub", callback=lambda: None, help="sub")] + + @click.group() + @click.pass_context + def root(ctx): + from deepctl_core.config import Config + ctx.obj = {"config": Config()} + + root.add_command(Group().get_click_group()) + # Not click.testing: on 3.14 it imports pdb, and with it asyncio and ssl. + try: + root.main(["grp", "sub", "--help"], prog_name="dg") + except SystemExit as exc: + assert exc.code == 0, exc.code + print(json.dumps(sorted(sys.modules))) + """, + home, + ) + for name in ("httpx", "keyring", "ssl", "deepgram"): + assert name not in modules, name diff --git a/packages/deepctl-core/tests/unit/test_output_channels.py b/packages/deepctl-core/tests/unit/test_output_channels.py index 8d9bb851..a1545daf 100644 --- a/packages/deepctl-core/tests/unit/test_output_channels.py +++ b/packages/deepctl-core/tests/unit/test_output_channels.py @@ -218,7 +218,8 @@ def test_explicit_flags_leave_stdout_parseable(self, capsys) -> None: payload = json.loads(captured.out) assert payload["status"] == "success" assert "Using credentials from" in captured.err - assert "No project ID specified" in captured.err + # Only commands that need a project say it's missing. + assert "No project ID specified" not in captured.err assert "Using credentials from" not in captured.out def test_project_id_line_also_goes_to_stderr(self, capsys) -> None: diff --git a/src/deepctl/_cli.py b/src/deepctl/_cli.py index 735e19ab..d547f247 100644 --- a/src/deepctl/_cli.py +++ b/src/deepctl/_cli.py @@ -826,6 +826,12 @@ def cli( # came from the command line, so it is never saved and gets # credentials only if the host is trusted. ctx.ensure_object(dict) + # An unexpanded ${...} in a DEEPGRAM_* variable counts as unset + # (W-ENV-PLACEHOLDER, printed once the output mode is known). + from deepctl_core.credentials import drop_env_placeholders + + drop_env_placeholders() + from deepctl_core import Config # Help from a spec runs no command: only the files' checks and diff --git a/tests/unit/test_command_imports.py b/tests/unit/test_command_imports.py index cbc82828..96c492a3 100644 --- a/tests/unit/test_command_imports.py +++ b/tests/unit/test_command_imports.py @@ -91,19 +91,7 @@ def _is_group(path: str) -> bool: @pytest.mark.parametrize( "path", [ - pytest.param( - p.split(" "), - marks=pytest.mark.xfail( - strict=True, - reason=( - "a subcommand group's help takes the full path, where the " - "parent group's callback builds an API client (httpx, ssl, " - "keyring) before the subcommand parses --help" - ), - ) - if _is_group(p) - else (), - ) + p.split(" ") for p in sorted(_manifest.commands()) if " " in p ], From 64399e6ccc3c344aa65a8cec0149292e7cc0233f Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:46:36 -0400 Subject: [PATCH 2/8] fix(core): send API requests through one transport deepctl_core.transport.Transport owns where API requests go and what they carry. The SDK facade builds its client through it. REST and WebSocket base URLs come only from check_credential_host (http -> ws, https -> wss), and Authorization is attached only after that check. User-visible changes: - One redirect refusal for REST and WebSocket: a 3xx is transport.RedirectRefused, the coded network_unavailable error (reason "unexpected redirect") with 0.4.1's text, which names both origins and says nothing was sent there (or no audio/text was sent, for a socket). for_status() maps every 3xx to it, so the client's own redirect check (_failed) and 0.4.1's separate ClickException go; deepctl_core.websocket re-exports the error and stays the only WebSocket open. Its message and exit_code read as before. Redirects were already refused; neither the key nor an upload body reaches another origin. - A connection reset or closed after the request started (a read or write error, or the server disconnecting without a response) is network_unavailable with E-NETWORK-DROPPED ("dropped before the response was complete"). A reset while connecting, DNS and TLS failures keep E-NETWORK. - DEEPGRAM_CLI_INSECURE now applies to the SDK's REST requests too, with 0.4.1's rule: only 1 or true count, only localhost, 127.0.0.1 and [::1] skip certificate verification, every other host is verified (with one W-INSECURE-REMOTE warning), and AI agent sessions ignore it. The login client uses the same rule. WebSocket streaming, dg api and debug probe always verify. - A WebSocket upgrade refused with an HTTP status maps through the same table as REST; DNS, connect and TLS failures map to network_unavailable, and an open that times out maps to timeout. - The shared client keeps no cookies. A backend seam (RestBackend by default) is where #142's SageMaker backend plugs in. shared_client, map_exception and raise_for stay module-level with unchanged signatures. dg whoami takes its default host from hosts.DEFAULT_BASE_URL. dg api takes its base URL from the Transport's host check and sends its request on the process's client, so a dg api call opens one connection like every other command. It still verifies TLS for every host, as in 0.4.1: when a person has DEEPGRAM_CLI_INSECURE set, dg api uses a separate always-verified client, and a TLS failure on a loopback host then says "DEEPGRAM_CLI_INSECURE doesn't apply to dg api, which always verifies TLS certificates." It asks for no redirects, and keeps 0.4.1's host scoping for the key. --- .../src/deepctl_cmd_api/command.py | 55 +- .../src/deepctl_cmd_api/messages.py | 6 + .../tests/unit/test_api_command.py | 84 ++- .../tests/unit/test_api_host_scope.py | 16 +- .../src/deepctl_cmd_listen/command.py | 23 +- .../tests/unit/test_live_redirect.py | 26 +- .../src/deepctl_cmd_login/command.py | 6 +- .../src/deepctl_cmd_speak/command.py | 19 +- .../tests/unit/test_speak_command.py | 41 ++ .../deepctl-core/src/deepctl_core/auth.py | 5 +- .../deepctl-core/src/deepctl_core/client.py | 137 ++--- .../src/deepctl_core/transport.py | 540 +++++++++++++++++- .../src/deepctl_core/websocket.py | 62 +- packages/deepctl-core/tests/unit/test_auth.py | 9 +- .../tests/unit/test_auth_insecure.py | 84 +++ .../deepctl-core/tests/unit/test_client.py | 9 +- .../deepctl-core/tests/unit/test_transport.py | 495 +++++++++++++++- .../tests/unit/test_ws_handshake_errors.py | 104 ++++ 18 files changed, 1490 insertions(+), 231 deletions(-) create mode 100644 packages/deepctl-core/tests/unit/test_ws_handshake_errors.py diff --git a/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py b/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py index 4792f874..43a434f9 100644 --- a/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py +++ b/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py @@ -1,6 +1,7 @@ """API command for deepctl.""" import json +import os import shutil import subprocess import sys @@ -288,14 +289,22 @@ def _build_headers( custom_headers: tuple[str, ...] | list[str] | None, ) -> dict[str, str]: """Build request headers with auth (when given) and custom headers.""" + from deepctl_core.transport import request_headers + headers: dict[str, str] = {} if api_key: - headers["Authorization"] = f"Token {api_key}" + # The key's host passed the check; the session ID goes only + # where the key goes. + headers = request_headers(api_key) if custom_headers: for header in custom_headers: if ":" in header: key, value = header.split(":", 1) - headers[key.strip()] = value.strip() + key = key.strip() + # A -H header replaces one of the same name, in any case. + for name in [k for k in headers if k.lower() == key.lower()]: + del headers[name] + headers[key] = value.strip() return headers def _build_body( @@ -363,6 +372,25 @@ def _apply_jq_filter(self, response_text: str, jq_filter: str) -> str: raise RuntimeError(f"jq error: {result.stderr.strip()}") return result.stdout.strip() + @staticmethod + def _insecure_ignored(url: str, error: BaseException) -> bool: + """A TLS failure on a loopback host while DEEPGRAM_CLI_INSECURE would + have skipped verification for the SDK: say it doesn't reach dg api.""" + from deepctl_core import auth, output, transport + + if output.is_ai_agent(): + return False + value = os.environ.get("DEEPGRAM_CLI_INSECURE", "").strip().lower() + origin = parse_origin(url) + if value not in ("1", "true") or origin is None: + return False + if origin.host not in auth._INSECURE_HOSTS: + return False + mapped = transport.map_exception(error) + return ( + mapped is not None and mapped.details.get("reason") == transport.REASON_TLS + ) + def handle( self, config: Config, @@ -382,9 +410,14 @@ def handle( raw_output = kwargs.get("raw", False) allow_host = kwargs.get("allow_foreign_host") - # Resolve URL against the profile's base URL. check_base_url raises - # (already reported) when that host isn't trusted with the key. - base_url = auth_manager.check_base_url() + from deepctl_core import transport + + # Resolve URL against the profile's base URL. The Transport's host + # check raises (already reported) when that host isn't trusted with + # the key. + base_url = transport.Transport( + config, check_host=auth_manager.check_base_url + ).rest_base() url = self._resolve_url(endpoint, base_url) attach, refusal = self._authorize(url, base_url, config, allow_host, method) @@ -432,11 +465,17 @@ def handle( # Make request try: - # Never follow redirects: a 3xx could point the key elsewhere. - with httpx.Client(timeout=30.0, follow_redirects=False) as http_client: - response = http_client.request(**request_kwargs) + # The process's one client (one connection), with TLS verified + # for every host as in 0.4.1: DEEPGRAM_CLI_INSECURE doesn't reach + # dg api. Never follow redirects: a 3xx could point the key + # elsewhere. + response = transport.verified_client().request( + **request_kwargs, follow_redirects=False, timeout=30.0 + ) except httpx.RequestError as e: console.print(f"[red]Request failed: {e}[/red]") + if self._insecure_ignored(url, e): + print_hint(msg.H_API_INSECURE_IGNORED) return ApiResult( status="error", message=f"Request failed: {e}", diff --git a/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py b/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py index 24fabd51..79f3e184 100644 --- a/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py +++ b/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py @@ -29,3 +29,9 @@ "{origin} answered with a redirect to {location_origin}. dg api doesn't " "follow redirects." ) +# After a TLS failure on localhost, 127.0.0.1 or [::1] while a person has +# DEEPGRAM_CLI_INSECURE set: dg api always verifies TLS (as in 0.4.1). +H_API_INSECURE_IGNORED = ( + "DEEPGRAM_CLI_INSECURE doesn't apply to dg api, which always verifies TLS " + "certificates." +) diff --git a/packages/deepctl-cmd-api/tests/unit/test_api_command.py b/packages/deepctl-cmd-api/tests/unit/test_api_command.py index 046c3447..c6542cdf 100644 --- a/packages/deepctl-cmd-api/tests/unit/test_api_command.py +++ b/packages/deepctl-cmd-api/tests/unit/test_api_command.py @@ -1,6 +1,8 @@ """Tests for API command.""" import json +import os +from pathlib import Path from unittest.mock import Mock, patch import httpx @@ -93,9 +95,22 @@ def test_build_headers(self, command): assert headers["Accept"] == "text/plain" def test_build_headers_no_custom(self, command): - """Test header building with only auth.""" + """With a key: the transport's request headers (identification, + session ID, auth), as every other API request sends.""" + from deepctl_core import transport + headers = command._build_headers("test-key", None) - assert headers == {"Authorization": "Token test-key"} + assert headers == transport.request_headers("test-key") + assert headers["Authorization"] == "Token test-key" + + def test_build_headers_without_a_key_has_no_session_id(self, command): + """No key (a host that doesn't get it): nothing from the builder.""" + assert command._build_headers(None, ["X-A: 1"]) == {"X-A": "1"} + + def test_custom_header_replaces_one_in_any_case(self, command): + headers = command._build_headers("test-key", ["user-agent: mine"]) + assert [k for k in headers if k.lower() == "user-agent"] == ["user-agent"] + assert headers["user-agent"] == "mine" def test_build_body_from_fields(self, command): """Test body building from field arguments.""" @@ -175,7 +190,7 @@ def test_apply_jq_filter_error(self, mock_run, mock_which, command): with pytest.raises(RuntimeError, match="jq error"): command._apply_jq_filter("{}", "invalid[") - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_success( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -187,8 +202,6 @@ def test_handle_success( mock_response.elapsed.total_seconds.return_value = 0.150 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -206,7 +219,7 @@ def test_handle_success( assert result.url == f"{DEFAULT_BASE_URL}/v1/projects" assert result.response_body == {"projects": []} - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_error_response( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -218,8 +231,6 @@ def test_handle_error_response( mock_response.elapsed.total_seconds.return_value = 0.050 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -234,14 +245,12 @@ def test_handle_error_response( assert result.status == "error" assert result.status_code == 404 - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_request_error( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): """Test handling of request errors.""" mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.side_effect = httpx.ConnectError("Connection refused") mock_client_class.return_value = mock_http @@ -275,7 +284,7 @@ def test_handle_no_api_key( assert result.status == "error" assert "No API key found" in result.message - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_post_with_fields( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -287,8 +296,6 @@ def test_handle_post_with_fields( mock_response.elapsed.total_seconds.return_value = 0.200 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -357,3 +364,52 @@ def test_api_result_serialization(self): data = result.model_dump() assert data["method"] == "POST" assert data["status_code"] == 201 + + +@pytest.mark.parametrize( + ("insecure", "host", "agent", "hinted"), + [ + ("1", "127.0.0.1", False, True), + ("true", "localhost", False, True), + ("1", "127.0.0.1", True, False), + ("0", "127.0.0.1", False, False), + ("1", "api.eu.deepgram.com", False, False), + ], +) +def test_insecure_hint_on_loopback_tls_failure(monkeypatch, capsys, insecure, host, agent, hinted): + """dg api always verifies TLS; say so when DEEPGRAM_CLI_INSECURE was set.""" + from deepctl_core import output + + base = f"https://{host}:8443" if host != "api.eu.deepgram.com" else f"https://{host}" + auth = Mock(spec=AuthManager) + auth.get_api_key.return_value = "test-api-key" + auth.check_base_url.return_value = base + config = Mock(spec=Config) + config.allowed_hosts.return_value = [f"{host}:8443"] + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", insecure) + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + monkeypatch.setattr(output.stderr_console, "_width", 400) + failing = Mock() + failing.request.side_effect = httpx.ConnectError( + "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed" + ) + monkeypatch.setattr("deepctl_core.transport.verified_client", lambda: failing) + result = ApiCommand().handle(config, auth, Mock(), endpoint="/v1/projects") + assert result.status == "error" + err = capsys.readouterr().err + assert ("DEEPGRAM_CLI_INSECURE doesn't apply to dg api," in err) is hinted + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_insecure_hint_matches_catalogue(): + + from deepctl_cmd_api import messages + + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + assert ( + catalogue["strings"]["H-API-INSECURE-IGNORED"] + == messages.H_API_INSECURE_IGNORED + ) diff --git a/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py b/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py index 0373dacf..4e247c8d 100644 --- a/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py +++ b/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py @@ -62,15 +62,17 @@ def handler(request: httpx.Request) -> httpx.Response: stream=httpx.ByteStream(b'{"ok": true}'), ) - real_client = httpx.Client + class Shared(httpx.Client): + """The shared client; `dg api` must ask for no redirects per request.""" - def client(**kwargs): - if "follow_redirects" not in kwargs: # not the `dg api` client - return real_client(**kwargs) - assert kwargs["follow_redirects"] is False - return real_client(transport=httpx.MockTransport(handler), **kwargs) + def request(self, *args, **kwargs): + assert kwargs["follow_redirects"] is False + return super().request(*args, **kwargs) - monkeypatch.setattr(api_module.httpx, "Client", client) + from deepctl_core import transport + + monkeypatch.setattr(transport, "_client", None) + transport._set_client(Shared(transport=httpx.MockTransport(handler))) return seen diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 93c7e3ba..cf91ff11 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -101,6 +101,14 @@ def _ws_base(client: DeepgramClient) -> str: return f"{'ws' if scheme.lower() in ('http', 'ws') else 'wss'}://{rest}" +def _socket_headers(api_key: str | None) -> dict[str, str]: + """What a live stream's WebSocket open carries: the transport's request + headers. The host was checked when the URL was built (``_ws_base``).""" + from deepctl_core import transport + + return transport.request_headers(api_key or "") + + # asyncio (and with it ssl) loads only when a live stream starts, so # `dg listen --help` and the prerecorded path don't import it. # The live engine imports asyncio, so it loads lazily too. @@ -852,7 +860,16 @@ def _prerecorded( # Raise (not return an error result) so an API rejection — e.g. a # --redact value the v1 endpoint refuses — exits non-zero and is # visible, rather than printing nothing and exiting 0. - detail = scrub_url_text(e, source) if url_like else e + from deepctl_core.output import print_debug + from deepctl_core.transport import plain_detail + + # The cause in its own words, never the SDK error's field dump; + # the whole exception only with --verbose. + raw = plain_detail(e) + detail = scrub_url_text(raw, source) if url_like else raw + print_debug( + escape(scrub_url_text(repr(e), source) if url_like else repr(e)) + ) raise click.ClickException(f"Transcription failed: {detail}") # ── Format transcript ────────────────────────────────────────── @@ -1081,7 +1098,7 @@ def on_message(raw: str | bytes) -> None: try: outcome = await live.run_session( url, - {"Authorization": f"Token {api_key}"}, + _socket_headers(api_key), on_message=on_message, queue=queue, closing=closing, @@ -1263,7 +1280,7 @@ def on_message(raw: str | bytes) -> None: outcome = await live.run_session( url, - {"Authorization": f"Token {api_key}"}, + _socket_headers(api_key), on_message=on_message, queue=queue, closing=closing, diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py b/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py index db905f42..3ff5ac3e 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py @@ -14,9 +14,9 @@ from dataclasses import dataclass, field from unittest.mock import MagicMock -import click import pytest from deepctl_cmd_listen.command import ListenCommand +from deepctl_core.websocket import RedirectRefused FAKE_KEY = "fa1ce0000000000000000000000000000000000d" @@ -157,7 +157,7 @@ def test_cross_origin_redirect_is_refused(mode, no_audio_devices): assert f"Token {FAKE_KEY}".encode() in servers.origin.requests[0] # Nothing at all reached the redirect target. assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.exit_code != 0 target = f"ws://127.0.0.1:{servers.target.port}" assert err.message == ( @@ -174,7 +174,7 @@ def test_same_origin_redirect_is_refused(mode, no_audio_devices): # The redirect back to the same host is not followed either. assert len(servers.origin.requests) == 1 assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.exit_code != 0 assert "doesn't follow redirects" in err.message @@ -185,7 +185,7 @@ def test_relative_location_names_the_same_origin(no_audio_devices): origin = f"ws://127.0.0.1:{servers.origin.port}" assert len(servers.origin.requests) == 1 - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message.startswith(f"{origin} answered with a redirect to {origin}.") @@ -194,7 +194,7 @@ def test_control_bytes_in_location_are_not_echoed(no_audio_devices): with _Servers("ws://evil\x9b31m.example/x") as servers: err = _run("stdin", servers.origin.origin) - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message.isascii() assert all(c.isprintable() for c in err.message) assert "redirect to ws://evil. " in err.message @@ -210,8 +210,22 @@ def test_unbalanced_bracket_in_location_gives_the_redirect_message( err = _run(mode, servers.origin.origin) assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message == ( f"ws://127.0.0.1:{servers.origin.port} answered with a redirect to ?. " "dg listen doesn't follow redirects, so no audio was sent." ) + + +@pytest.mark.parametrize("mode", ["mic", "stdin"]) +def test_upgrade_carries_the_request_headers(mode, no_audio_devices): + """The live upgrade identifies the run as every other API request does: + the SDK's User-Agent, the session ID and the key.""" + from deepctl_core import transport + + with _Servers(CROSS) as servers: + _run(mode, servers.origin.origin) + + request = servers.origin.requests[0].decode("latin-1").lower() + for name, value in transport.request_headers(FAKE_KEY).items(): + assert f"\r\n{name.lower()}: {value.lower()}\r\n" in request, name diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index fe486e01..24d26abd 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1647,7 +1647,9 @@ def handle( masked = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****" project_id = auth_manager.get_project_id() - base_url = profile_cfg.base_url or "https://api.deepgram.com" + from deepctl_core.hosts import DEFAULT_BASE_URL + + base_url = profile_cfg.base_url or DEFAULT_BASE_URL # Render the human block only in default mode. For json/yaml/csv the # framework serialises the returned result to stdout, so printing the @@ -1666,7 +1668,7 @@ def handle( if project_id else " Project ID: [dim]not set[/dim]" ) - if base_url != "https://api.deepgram.com": + if base_url != DEFAULT_BASE_URL: console.print(f" Base URL: {base_url}") if profile_cfg.dg_token_expires_at and has_profile_key: console.print(f" Expires: {profile_cfg.dg_token_expires_at}") diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index be8da794..cc547559 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -294,6 +294,19 @@ def _os_error_text(exc: OSError) -> str: _CLOSE_TIMEOUT_S = 2.0 +def _failed(what: str, exc: BaseException) -> click.ClickException: + """``{what}: {cause}`` for an error no coded message covers. + + The cause is the underlying error's own message, never the SDK error's + field dump; the whole exception goes to --verbose output only. + """ + from deepctl_core.output import print_debug + from deepctl_core.transport import plain_detail + + print_debug(escape(f"{what}: {exc!r}")) + return click.ClickException(f"{what}: {plain_detail(exc)}") + + def _close(*streams: object, timeout: float = _CLOSE_TIMEOUT_S) -> None: """Close audio iterators (and the responses behind them), quietly and in order, and wait at most ``timeout`` seconds for all of them together.""" @@ -1431,7 +1444,7 @@ def _speak_flux( except _SINK_ERRORS: raise except Exception as e: - raise click.ClickException(f"Flux streaming failed: {e}") + raise _failed("Flux streaming failed", e) from e finally: _close(stream, raw) @@ -1609,7 +1622,7 @@ def _speak_v1( sample_rate=sample_rate, ) except Exception as e: - raise click.ClickException(f"Error generating speech: {e}") + raise _failed("Error generating speech", e) from e try: if player in _STDIN_PLAYER_ARGV: @@ -1658,7 +1671,7 @@ def _speak_v1( # in every output mode — a returned BaseResult is only printed and # still exits 0. Reachable now that unknown models (e.g. a bare # "flux" typo) route here instead of the raising v2 path. - raise click.ClickException(f"Error generating speech: {e}") + raise _failed("Error generating speech", e) from e finally: _close(audio_iter) diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py index bc03c6e9..b513fd84 100644 --- a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py @@ -802,6 +802,47 @@ def _boom(): file=None, ) + @patch("deepctl_cmd_speak.command.sys") + def test_flux_failure_shows_the_cause_not_the_sdk_fields( + self, + mock_sys, + command, + mock_config, + mock_auth_manager, + mock_client, + tmp_path, + ): + """The client's ApiError wrapper shows as its cause's own words.""" + from deepgram.core.api_error import ApiError + + mock_sys.stdin.isatty.return_value = True + mock_sys.stdout.isatty.return_value = True + + def _boom(): + try: + raise RuntimeError("socket closed") + except RuntimeError as e: + raise ApiError(body=f"Text-to-speech streaming failed: {e}") + yield # pragma: no cover — make this a generator + + mock_client.speak_text_stream.return_value = _boom() + + with pytest.raises(click.ClickException) as err: + command.handle( + config=mock_config, + auth_manager=mock_auth_manager, + client=mock_client, + text="Hello from Flux", + output=str(tmp_path / "x.wav"), + model="flux-alexis-en", + encoding=None, + container=None, + sample_rate=None, + file=None, + ) + assert err.value.message == "Flux streaming failed: socket closed" + assert "status_code" not in err.value.message + @patch("deepctl_cmd_speak.command.sys") def test_handle_flux_streams_to_stdout_incrementally( self, mock_sys, command, mock_config, mock_auth_manager, mock_client diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index 2463a8d6..ac1506dc 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -167,6 +167,7 @@ def _insecure_mounts() -> "dict[str, httpx.BaseTransport]": httpx builds from HTTPS_PROXY/ALL_PROXY/NO_PROXY, so every other host keeps both certificate verification and the proxy settings. """ + _load_modules("httpx") unverified = httpx.HTTPTransport(verify=False) return { f"all://{'[::1]' if host == '::1' else host}": unverified @@ -514,7 +515,7 @@ def verify_api_key( try: headers = { - "Authorization": f"Token {api_key}", + **transport.request_headers(api_key), "Content-Type": "application/json", } @@ -587,7 +588,7 @@ def verify_credentials( # Make API request to verify both credentials try: headers = { - "Authorization": f"Token {api_key}", + **transport.request_headers(api_key), "Content-Type": "application/json", } diff --git a/packages/deepctl-core/src/deepctl_core/client.py b/packages/deepctl-core/src/deepctl_core/client.py index ba86112e..70515bbe 100644 --- a/packages/deepctl-core/src/deepctl_core/client.py +++ b/packages/deepctl-core/src/deepctl_core/client.py @@ -7,8 +7,7 @@ from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlencode -from . import transport -from .hosts import DEFAULT_BASE_URL +from . import hosts, transport from .output import stderr_console if TYPE_CHECKING: @@ -16,8 +15,7 @@ from datetime import datetime # Bound at run time by _sdk() below, not by these imports. - from deepgram import DeepgramClient as DGClient # noqa: TC004 - from deepgram import DeepgramClientEnvironment # noqa: TC004 + from deepgram import DeepgramClient as DGClient from deepgram.core.api_error import ApiError # noqa: TC004 from .auth import AuthManager @@ -90,8 +88,9 @@ def __init__(self, config: Config, auth_manager: AuthManager): self.auth_manager = auth_manager self._client: DGClient | None = None self._project_id: str | None = None - # The REST base the SDK client was given (for redirect errors). - self._rest_base: str = DEFAULT_BASE_URL + self.transport = transport.Transport( + config, check_host=auth_manager.check_base_url + ) # Kept out of error text. self._secrets: tuple[str, ...] = () @@ -122,78 +121,26 @@ def _create_client(self) -> DGClient: if not api_key: raise ApiError(body="No API key available") - # The SDK sends the key to whatever environment it is given, so the - # base URL goes through the same host check as every other path. - base_url = self.auth_manager.check_base_url() - try: - # One connection pool per process; requests reuse its connection. - kwargs: dict[str, Any] = { - "api_key": api_key, - "httpx_client": transport.shared_client(), - } - - if base_url and base_url != DEFAULT_BASE_URL: - # Derive both the REST and WebSocket endpoints from the - # configured base URL, so a custom base works for batch REST - # *and* streaming. Preserve the transport implied by the input - # scheme (http→ws, https→wss) so a plaintext local/staging - # endpoint isn't forced onto TLS. agent_rest is required by the - # SDK environment. - rest_scheme, ws_scheme, host = _split_base_url(base_url) - self._rest_base = f"{rest_scheme}://{host}" - kwargs["environment"] = DeepgramClientEnvironment( - base=f"{rest_scheme}://{host}", - production=f"{ws_scheme}://{host}", - agent=f"{ws_scheme}://{host}", - agent_rest=f"{rest_scheme}://{host}", - ) - - # A redirect would take the request body (audio, text) to - # another host, and with the SDK's default the key too on a - # same-origin hop. Refuse it; _failed() words the error. - kwargs["follow_redirects"] = False - - client = DGClient(**kwargs) + # The Transport checks the host before the SDK gets the key, and + # derives the REST and WebSocket endpoints from the same base. + client = cast("DGClient", self.transport.sdk_client(api_key)) self._project_id = project_id self._secrets = (api_key,) return client + except hosts.CredentialHostError: + raise except Exception as e: console.print(f"[red]Error creating Deepgram client:[/red] {e}") raise ApiError(body=f"Failed to create client: {e}") - def _failed(self, what: str, e: Exception) -> Exception: - """The error to raise for a failed SDK call. - - A refused redirect passes through unchanged, and a 3xx answer - (the SDK client doesn't follow redirects) becomes the same refusal. - Anything else is wrapped as before. - """ - from .websocket import RedirectRefused, redirect_refused - - if isinstance(e, RedirectRefused): - return e - status = getattr(e, "status_code", None) - if isinstance(e, ApiError) and status is not None and 300 <= status < 400: - headers = {k.lower(): v for k, v in (e.headers or {}).items()} - return redirect_refused( - self._rest_base, headers.get("location"), command="dg", payload=None - ) - return ApiError(body=f"{what}: {e}") - def _mapped(self, chunks: Iterator[bytes]) -> Iterator[bytes]: - """Yield ``chunks``; a request failure while reading them is mapped, - and a 3xx answer becomes the redirect refusal.""" - from .websocket import RedirectRefused - + """Yield ``chunks``; a request failure while reading them is mapped.""" try: yield from chunks except Exception as e: transport.raise_for(e, self._secrets) - refused = self._failed("", e) - if isinstance(refused, RedirectRefused): - raise refused from e raise # ── Speech-to-Text (pre-recorded) ────────────────────────────── @@ -246,7 +193,7 @@ def transcribe_file( raise except Exception as e: transport.raise_for(e, self._secrets, timeout_s=timeout_s) - raise self._failed("Transcription failed", e) + raise ApiError(body=f"Transcription failed: {e}") def transcribe_url( self, @@ -281,7 +228,7 @@ def transcribe_url( except Exception as e: transport.raise_for(e, self._secrets, timeout_s=timeout_s) - raise self._failed("Transcription failed", e) + raise ApiError(body=f"Transcription failed: {e}") # ── Text-to-Speech ───────────────────────────────────────────── @@ -311,7 +258,7 @@ def speak_text( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text-to-speech failed", e) + raise ApiError(body=f"Text-to-speech failed: {e}") def speak_text_stream( self, @@ -353,7 +300,7 @@ def speak_text_stream( # its connect sent them; creating the client runs the key and host # checks. headers = dict(self.client._client_wrapper.get_headers()) - url = f"{self.ws_base_url()}/v2/speak?{urlencode(query)}" + url = f"{self.transport.ws_base().rstrip('/')}/v2/speak?{urlencode(query)}" try: with connect_sync( url, @@ -372,15 +319,7 @@ def speak_text_stream( yield message except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text-to-speech streaming failed", e) - - def ws_base_url(self) -> str: - """The WebSocket base for the checked base URL, without a slash.""" - base_url = self.auth_manager.check_base_url() - if not base_url or base_url == DEFAULT_BASE_URL: - base_url = DEFAULT_BASE_URL - _rest, ws_scheme, host = _split_base_url(str(base_url)) - return f"{ws_scheme}://{host}" + raise ApiError(body=f"Text-to-speech streaming failed: {e}") # ── Text Intelligence (Read) ─────────────────────────────────── @@ -415,7 +354,7 @@ def analyze_text( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text analysis failed", e) + raise ApiError(body=f"Text analysis failed: {e}") # ── Models ───────────────────────────────────────────────────── @@ -431,7 +370,7 @@ def list_models(self, include_outdated: bool = False) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list models", e) + raise ApiError(body=f"Failed to list models: {e}") def get_model(self, model_id: str) -> dict[str, Any]: """Get a specific model by ID.""" @@ -441,7 +380,7 @@ def get_model(self, model_id: str) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get model", e) + raise ApiError(body=f"Failed to get model: {e}") # ── Projects ─────────────────────────────────────────────────── @@ -453,7 +392,7 @@ def get_projects(self) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get projects", e) + raise ApiError(body=f"Failed to get projects: {e}") def get_project(self, project_id: str | None = None) -> dict[str, Any]: """Get specific project.""" @@ -464,7 +403,7 @@ def get_project(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get project", e) + raise ApiError(body=f"Failed to get project: {e}") def create_project(self, name: str, company: str | None = None) -> dict[str, Any]: """Create a new project.""" @@ -478,7 +417,7 @@ def create_project(self, name: str, company: str | None = None) -> dict[str, Any except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create project", e) + raise ApiError(body=f"Failed to create project: {e}") # ── Usage ────────────────────────────────────────────────────── @@ -504,7 +443,7 @@ def get_usage( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get usage", e) + raise ApiError(body=f"Failed to get usage: {e}") # ── API Keys ─────────────────────────────────────────────────── @@ -525,7 +464,7 @@ def list_keys( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list keys", e) + raise ApiError(body=f"Failed to list keys: {e}") def create_key( self, @@ -557,7 +496,7 @@ def create_key( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create key", e) + raise ApiError(body=f"Failed to create key: {e}") def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: """Get a specific API key.""" @@ -568,7 +507,7 @@ def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get key", e) + raise ApiError(body=f"Failed to get key: {e}") def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: """Delete an API key.""" @@ -579,7 +518,7 @@ def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, An except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to delete key", e) + raise ApiError(body=f"Failed to delete key: {e}") # ── Requests ─────────────────────────────────────────────────── @@ -618,7 +557,7 @@ def list_requests( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list requests", e) + raise ApiError(body=f"Failed to list requests: {e}") def get_request( self, request_id: str, project_id: str | None = None @@ -631,7 +570,7 @@ def get_request( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get request", e) + raise ApiError(body=f"Failed to get request: {e}") # ── Billing ──────────────────────────────────────────────────── @@ -660,7 +599,7 @@ def get_billing_breakdown( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get billing breakdown", e) + raise ApiError(body=f"Failed to get billing breakdown: {e}") def get_balances(self, project_id: str | None = None) -> dict[str, Any]: """Get project balances.""" @@ -671,7 +610,7 @@ def get_balances(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get balances", e) + raise ApiError(body=f"Failed to get balances: {e}") # ── Members ──────────────────────────────────────────────────── @@ -684,7 +623,7 @@ def list_members(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list members", e) + raise ApiError(body=f"Failed to list members: {e}") def remove_member( self, member_id: str, project_id: str | None = None @@ -697,7 +636,7 @@ def remove_member( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to remove member", e) + raise ApiError(body=f"Failed to remove member: {e}") def list_member_scopes( self, member_id: str, project_id: str | None = None @@ -712,7 +651,7 @@ def list_member_scopes( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list member scopes", e) + raise ApiError(body=f"Failed to list member scopes: {e}") def update_member_scopes( self, member_id: str, scope: str, project_id: str | None = None @@ -727,7 +666,7 @@ def update_member_scopes( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to update member scopes", e) + raise ApiError(body=f"Failed to update member scopes: {e}") # ── Invites ──────────────────────────────────────────────────── @@ -740,7 +679,7 @@ def list_invites(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list invites", e) + raise ApiError(body=f"Failed to list invites: {e}") def create_invite( self, email: str, scope: str, project_id: str | None = None @@ -755,7 +694,7 @@ def create_invite( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create invite", e) + raise ApiError(body=f"Failed to create invite: {e}") def delete_invite( self, email: str, project_id: str | None = None @@ -768,7 +707,7 @@ def delete_invite( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to delete invite", e) + raise ApiError(body=f"Failed to delete invite: {e}") # ── Utilities ────────────────────────────────────────────────── diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py index d4ec6a16..20ae613c 100644 --- a/packages/deepctl-core/src/deepctl_core/transport.py +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -1,19 +1,27 @@ -"""One HTTP client per process, and the map from transport failures to codes. - -Every REST call the SDK facade makes goes through :func:`shared_client`: one -``httpx.Client`` (one connection pool) per process, created on first use so -``--version`` and ``--help`` never import httpx. The client carries no -credentials and no base URL. The key and the host still come from -``DeepgramClient``, which takes the base URL from -:func:`deepctl_core.hosts.check_credential_host` before any request carries -the key; this module never decides where a credential may go. +"""Where API requests go, what they carry, and what their failures mean. + +:class:`Transport` owns the REST and WebSocket base URLs, which come only +from :func:`deepctl_core.hosts.check_credential_host` (``http`` -> ``ws``, +``https`` -> ``wss``), and attaches ``Authorization`` only after that check +has passed for the exact origin. Requests go through :func:`shared_client`: +one ``httpx.Client`` (one connection pool) per process, created on first use +so ``--version`` and ``--help`` never import httpx. It follows no redirects +(a 3xx with credentials is an error, never a second request), keeps no +cookies, and verifies TLS. A person's ``DEEPGRAM_CLI_INSECURE=1`` (or +``true``) skips verification for localhost, 127.0.0.1 and [::1] only, and +never under an AI agent (``auth._insecure_allowed`` and +``auth._insecure_mounts``, as in 0.4.2). :func:`verified_client` is the +always-verified client for ``dg api``; WebSocket streams always verify. +The SDK facade builds its client through :attr:`Transport.backend`, a seam +for other backends (#142's SageMaker), with the REST backend as the default. :func:`raise_for` turns a failed request into a :class:`TransportError` with a catalogue code (``transport_mapping`` in ``error_codes.json``, in -order): DNS, connect, TLS, reset or a truncated 2xx body -> +order): DNS, connect, TLS, reset, a truncated 2xx body or a 3xx -> ``network_unavailable``; timeout -> ``timeout``; 401 -> ``auth_invalid``; 403 -> ``auth_forbidden``; 404 -> ``not_found``; 429 -> ``rate_limited``; 400, 422 and any other 4xx -> ``invalid_argument``; 5xx -> ``server_error``. +A WebSocket upgrade refused with an HTTP status maps through the same table. ``BaseCommand.execute`` renders it: a coded result on stdout under a machine format, and two stderr lines. """ @@ -31,15 +39,22 @@ from . import credentials if TYPE_CHECKING: + from collections.abc import Callable + import httpx from .auth import AuthManager + from .config import Config # The SDK's own default for a client it builds itself. DEFAULT_TIMEOUT_S = 60.0 _lock = threading.Lock() _client: httpx.Client | None = None +# True when _client has DEEPGRAM_CLI_INSECURE's unverified loopback mounts. +_client_insecure = False +# The always-verified client for dg api, when _client isn't one. +_verified: httpx.Client | None = None # Where the most recent request went, for the error text: method, host and # path only (never the query string, which can carry values from argv). _last_request: dict[str, str] = {} @@ -48,12 +63,18 @@ def _record_request(request: httpx.Request) -> None: _last_request.clear() _last_request.update( - method=request.method, host=request.url.host, path=request.url.path + method=request.method, + host=request.url.host, + path=request.url.path, + origin=f"{request.url.scheme}://{request.url.netloc.decode('ascii')}", ) def _install(client: httpx.Client) -> httpx.Client: _no_cookies(client) + # A redirect could take the key, or a request body, to another origin + # a 3xx is an error (see map_exception), never followed. + client.follow_redirects = False hooks = client.event_hooks hooks.setdefault("request", []) if _record_request not in hooks["request"]: @@ -62,23 +83,58 @@ def _install(client: httpx.Client) -> httpx.Client: return client +def _new_client(*, insecure_loopback: bool) -> httpx.Client: + import httpx + + from .auth import _insecure_mounts, _warn_if_verified + + if insecure_loopback: + # As AuthManager.client (0.4.2): verification stays on, and only the + # loopback hosts get unverified transports. + return httpx.Client( + timeout=DEFAULT_TIMEOUT_S, + follow_redirects=False, + verify=True, + mounts=_insecure_mounts(), + event_hooks={"request": [_warn_if_verified]}, + ) + return httpx.Client(timeout=DEFAULT_TIMEOUT_S, follow_redirects=False) + + def shared_client() -> httpx.Client: """The process's one ``httpx.Client``, created on first use. - It follows no redirects: a 3xx could take the key, or a request body, - to another host (see ``DeepgramClient._failed``). + It follows no redirects, keeps no cookies and verifies TLS. A person's + ``DEEPGRAM_CLI_INSECURE=1`` (or ``true``) skips verification for + localhost, 127.0.0.1 and [::1] only; never under an AI agent + (``auth._insecure_allowed``, ``auth._insecure_mounts``). """ - global _client + global _client, _client_insecure with _lock: if _client is None: - import httpx + from .auth import _insecure_allowed - _client = _install( - httpx.Client(timeout=DEFAULT_TIMEOUT_S, follow_redirects=False) - ) + _client_insecure = _insecure_allowed() + _client = _install(_new_client(insecure_loopback=_client_insecure)) return _client +def verified_client() -> httpx.Client: + """A client that verifies TLS for every host (``dg api``, as in 0.4.1). + + The shared client itself, unless ``DEEPGRAM_CLI_INSECURE`` gave it + unverified loopback transports; then a second client that has none. + """ + global _verified + client = shared_client() + with _lock: + if not _client_insecure: + return client + if _verified is None: + _verified = _install(_new_client(insecure_loopback=False)) + return _verified + + def _no_cookies(client: httpx.Client) -> None: """Keep no cookies: one process can serve several profiles and hosts.""" from http.cookiejar import DefaultCookiePolicy @@ -88,12 +144,188 @@ def _no_cookies(client: httpx.Client) -> None: def _set_client(client: httpx.Client | None) -> None: """Replace the shared client (tests pass one with an ``httpx.MockTransport``).""" - global _client + global _client, _client_insecure, _verified with _lock: _client = _install(client) if client is not None else None + _client_insecure = False + _verified = None _last_request.clear() +# ── Base URLs and credentials ──────────────────────────────────────── + + +def ws_url(base_url: str) -> str: + """The WebSocket form of a REST base URL: ``http`` -> ``ws``, + ``https`` (or none) -> ``wss``; host, port and path unchanged.""" + scheme, sep, rest = base_url.partition("://") + if not sep: + return f"wss://{base_url}" + scheme = scheme.lower() + if scheme in ("http", "ws"): + return f"ws://{rest}" + return f"wss://{rest}" + + +def rest_url(base_url: str) -> str: + """The REST form of a base URL: ``ws`` -> ``http``, ``wss`` (or none) -> + ``https``.""" + scheme, sep, rest = base_url.partition("://") + if not sep: + return f"https://{base_url}" + scheme = scheme.lower() + if scheme in ("http", "ws"): + return f"http://{rest}" + return f"https://{rest}" + + +def default_host() -> str: + """The Deepgram API host requests go to by default (``hosts.DEFAULT_BASE_URL``).""" + from urllib.parse import urlsplit + + from .hosts import DEFAULT_BASE_URL + + return urlsplit(DEFAULT_BASE_URL).hostname or DEFAULT_BASE_URL + + +# One random ID per run, sent as x-deepgram-session-id with every request +# that carries the key, so Deepgram can group a run's requests. It says +# nothing about the user or the machine. +_session_id: str | None = None + +SESSION_HEADER = "x-deepgram-session-id" + + +def session_id() -> str: + """This run's ``x-deepgram-session-id``, the same on every request.""" + import uuid + + global _session_id + if _session_id is None: + _session_id = str(uuid.uuid4()) + return _session_id + + +def request_headers(api_key: str) -> dict[str, str]: + """The headers every Deepgram API request and WebSocket open carries. + + The SDK's identification (``User-Agent: deepgram-sdk/`` and its + ``X-Fern-*`` headers, built as its ``get_headers`` builds them), this + run's session ID, and ``Authorization``. The SDK client gets the same + session ID (:class:`RestBackend`), so requests dg sends itself and + requests the SDK sends identify the run alike. Callers check the host + first; :meth:`Transport.auth_headers` does. + """ + import platform + from importlib import metadata + + headers: dict[str, str] = {} + try: + version = metadata.version("deepgram-sdk") + except metadata.PackageNotFoundError: + pass + else: + headers = { + "User-Agent": "deepgram-sdk/" + version, + "X-Fern-Language": "Python", + "X-Fern-Runtime": f"python/{platform.python_version()}", + "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", + "X-Fern-SDK-Name": "deepgram-sdk", + "X-Fern-SDK-Version": version, + } + headers[SESSION_HEADER] = session_id() + headers["Authorization"] = f"Token {api_key}" + return headers + + +class RestBackend: + """The default backend: the Deepgram SDK over the shared ``httpx.Client``. + + A backend turns a checked base URL and a key into the SDK client the + facade uses. #142's SageMaker backend plugs in here (R-16). + """ + + name = "rest" + + def sdk_client(self, api_key: str, rest_base: str, ws_base: str) -> Any: + # Through the facade module's lazily bound names (tests patch them). + from . import client as facade + from .hosts import DEFAULT_BASE_URL + + facade._sdk() + sdk: dict[str, Any] = vars(facade) + + kwargs: dict[str, Any] = { + "api_key": api_key, + # One connection pool per process; requests reuse its connection. + "httpx_client": shared_client(), + # The run's ID, so SDK requests match request_headers(). + "session_id": session_id(), + } + if rest_base != DEFAULT_BASE_URL: + # Both the REST and the WebSocket endpoints come from the checked + # base URL, so a custom base works for batch REST and streaming. + # agent_rest is required by the SDK environment. + kwargs["environment"] = sdk["DeepgramClientEnvironment"]( + base=rest_base, + production=ws_base, + agent=ws_base, + agent_rest=rest_base, + ) + return sdk["DGClient"](**kwargs) + + +class Transport: + """Where this run's API requests go and what they carry. + + Base URLs come from :func:`deepctl_core.hosts.check_credential_host` + only; it raises ``CredentialHostError`` (already reported) when the host + isn't trusted with credentials, before anything carries the key. + """ + + def __init__( + self, + config: Config, + backend: Any = None, + *, + check_host: Callable[[], str] | None = None, + ) -> None: + """``check_host`` is ``AuthManager.check_base_url`` when there is one + (the same ``check_credential_host`` call on the same config).""" + self.config = config + self.backend = backend if backend is not None else RestBackend() + self._check_host = check_host + + def rest_base(self) -> str: + """The checked REST base URL (scheme, host, port and path).""" + if self._check_host is not None: + return rest_url(self._check_host()) + from .hosts import check_credential_host + + return rest_url(check_credential_host(self.config)) + + def ws_base(self) -> str: + """The checked WebSocket base URL (``http`` -> ``ws``, ``https`` -> ``wss``).""" + return ws_url(self.rest_base()) + + def auth_headers(self, api_key: str) -> dict[str, str]: + """:func:`request_headers` for ``api_key`` (identification, session + ID and ``Authorization``), once the host check has passed.""" + self.rest_base() + return request_headers(api_key) + + @property + def client(self) -> httpx.Client: + """The process's shared ``httpx.Client``.""" + return shared_client() + + def sdk_client(self, api_key: str) -> Any: + """The SDK client for ``api_key``, built by the backend once the host + check has passed.""" + rest = self.rest_base() + return self.backend.sdk_client(api_key, rest, ws_url(rest)) + + # ── Catalogue text (error_codes.json "strings") ────────────────────── STRINGS: dict[str, str] = { @@ -162,6 +394,9 @@ def _set_client(client: httpx.Client | None) -> None: "Retry once. If it fails again, tell the user the connection to Deepgram " "keeps dropping." ), + "E-NETWORK-DROPPED": ( + "The connection to {host} dropped before the response was complete ({reason})." + ), "E-NETWORK-UNREADABLE": "Got a response from {host} that {prog} couldn't read.", "H-NETWORK-UNREADABLE": ( "A proxy, captive portal or firewall may be answering in Deepgram's " @@ -172,6 +407,20 @@ def _set_client(client: httpx.Client | None) -> None: "network (a proxy, captive portal or firewall) is answering in " "Deepgram's place." ), + # 0.4.1's redirect refusal (origins only); {sent} is "nothing was sent + # there" for REST and "no audio was sent" (or text) for a WebSocket. + "E-NETWORK-REDIRECT": ( + "{origin} answered with a redirect to {location_origin}. {command} " + "doesn't follow redirects, so {sent}." + ), + "H-NETWORK-REDIRECT": ( + "A network sign-in page or proxy may be redirecting requests. If {host} " + "is your own server, use the address it redirects to." + ), + "A-NETWORK-REDIRECT": ( + "Tell the user {host} redirected the request. Don't retry until they " + "have checked the network or the configured URL." + ), "E-TIMEOUT": "{host} didn't respond within {timeout_s} s.", "H-TIMEOUT": "Try again. Large files take longer to upload on slow connections.", "A-TIMEOUT": ( @@ -245,8 +494,15 @@ def _set_client(client: httpx.Client | None) -> None: "server_error": ("E-SERVER", "H-SERVER", "A-SERVER", None), } -# network_unavailable's other messages: the response was cut off, or it -# arrived complete but isn't Deepgram's (a proxy or captive portal page). +# network_unavailable's other messages: the connection dropped after the +# request started, the response was cut off, it arrived complete but isn't +# Deepgram's (a proxy or captive portal page), or it was a redirect. +NETWORK_DROPPED: Messages = ( + "E-NETWORK-DROPPED", + "H-NETWORK-CUT", + "A-NETWORK-CUT", + None, +) NETWORK_CUT: Messages = ("E-NETWORK-CUT", "H-NETWORK-CUT", "A-NETWORK-CUT", None) NETWORK_UNREADABLE: Messages = ( "E-NETWORK-UNREADABLE", @@ -254,6 +510,12 @@ def _set_client(client: httpx.Client | None) -> None: "A-NETWORK-UNREADABLE", None, ) +NETWORK_REDIRECT: Messages = ( + "E-NETWORK-REDIRECT", + "H-NETWORK-REDIRECT", + "A-NETWORK-REDIRECT", + None, +) # auth_required's other message: the only key was in a user config file this # AI agent session doesn't trust. next_step is null: a person @@ -274,6 +536,7 @@ def _set_client(client: httpx.Client | None) -> None: REASON_TRUNCATED = "response cut off" REASON_UNREADABLE = "unreadable response" REASON_PROXY = "the proxy refused the connection" +REASON_REDIRECT = "unexpected redirect" REASON_OTHER = "connection failed" KEY_SOURCE_ENV = credentials.KEY_SOURCE_ENV @@ -309,6 +572,66 @@ def error(self) -> str: return STRINGS[self.messages[0]].format(**self.values) +class RedirectRefused(TransportError): + """A request was answered with a redirect, which is never followed. + + REST and WebSocket alike: the key and the request's audio or text go + only to the host that passed the host check. ``message`` and + ``exit_code`` read as on the ``click.ClickException`` it was in 0.4.1. + """ + + exit_code = 1 + + @property + def message(self) -> str: + return self.error + + +def redirect_refused( + url: str, + location: str | None, + *, + command: str = "dg", + payload: str | None = None, +) -> RedirectRefused: + """The error for a redirect from ``url`` to ``location``. + + Only origins are shown, never paths or queries, and anything a server + put in the Location header is made safe to print first. ``payload`` + names what a WebSocket would have sent ("audio", "text"); ``None`` for + REST, where the request reached ``url`` and nothing went further. + """ + from urllib.parse import urljoin + + from .hosts import parse_origin + from .messages import display + + here = parse_origin(url) + there = None + if location: + # Header bytes outside ASCII arrive as surrogates; show them as "?". + location = location.encode("ascii", "replace").decode("ascii") + try: + there = parse_origin(urljoin(url, location)) + except ValueError: # e.g. an unbalanced "[" in the host + there = None + host = _clean(here.host if here else url) + sent = "nothing was sent there" if payload is None else f"no {payload} was sent" + return RedirectRefused( + "network_unavailable", + { + "origin": display(here.origin if here else "?"), + "location_origin": display(there.origin if there else "?"), + "command": command, + "sent": sent, + "host": host, + "prog": prog() or "dg", + }, + {"host": host, "reason": REASON_REDIRECT}, + NETWORK_REDIRECT, + ) + + # ── Mapping ────────────────────────────────────────────────────────── _SECRET_URL = re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]*@") @@ -441,14 +764,27 @@ def map_exception( if isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError)): host = _host(exc) reason = _network_reason(exc) - cut = reason == REASON_TRUNCATED + messages: Messages | None = None + if reason == REASON_TRUNCATED: + messages = NETWORK_CUT + elif reason in (REASON_RESET, REASON_CLOSED) and isinstance( + exc, (httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError) + ): + # The connection was open and the request had started, so + # "couldn't reach" would be wrong. A reset while connecting + # (ConnectError) stays E-NETWORK. + messages = NETWORK_DROPPED return TransportError( "network_unavailable", {"host": host, "reason": reason}, {"host": host, "reason": reason}, - NETWORK_CUT if cut else None, + messages, ) + ws = _map_websocket(exc, secrets) + if ws is not None: + return ws + status = getattr(exc, "status_code", None) if not isinstance(status, int) or "deepgram" not in sys.modules: return None @@ -456,7 +792,6 @@ def map_exception( if not isinstance(exc, ApiError): return None - headers = exc.headers if 200 <= status < 300: # The SDK got a complete success response it couldn't parse: most # often not Deepgram's at all (a proxy's or captive portal's page). @@ -468,14 +803,37 @@ def map_exception( {"host": host, "reason": REASON_UNREADABLE}, NETWORK_UNREADABLE, ) + return for_status(status, exc.headers, exc.body, secrets) + + +def for_status( + status: int, + headers: object = None, + body: object = None, + secrets: tuple[str, ...] = (), + *, + host: str | None = None, + path: str | None = None, +) -> TransportError | None: + """The coded error for an HTTP error status, REST or WebSocket upgrade. + + ``host`` and ``path`` name the request when it didn't go through the + shared client (a WebSocket upgrade); else the last request's are used. + """ + if 300 <= status < 400: + # Credentialed requests don't follow redirects. + origin = host or _last_request.get("origin") or _host() + return redirect_refused( + origin, _header(headers, "location"), command=_running_command() + ) if status == 401: return TransportError("auth_invalid", {}, {"http_status": 401}) if status == 403: return TransportError("auth_forbidden", {}, {"http_status": 403}) if status == 404: - path = _clean(_last_request.get("path") or "this request") + where = _clean(path or _last_request.get("path") or "this request") return TransportError( - "not_found", {"path": path}, {"http_status": 404, "path": path} + "not_found", {"path": where}, {"http_status": 404, "path": where} ) if status == 429: details: dict[str, Any] = {"http_status": 429} @@ -488,7 +846,7 @@ def map_exception( "invalid_argument", { "http_status": status, - "api_message": _api_message(exc.body, status, secrets), + "api_message": _api_message(body, status, secrets), }, {"http_status": status}, ) @@ -501,10 +859,138 @@ def map_exception( return None +# Seconds a WebSocket handshake may take before it maps to ``timeout``. +WS_OPEN_TIMEOUT_S = 10.0 + + +def _ws_status(exc: BaseException) -> tuple[int, object, object] | None: + """(status, headers, body) of a refused WebSocket upgrade, if ``exc`` is one.""" + if "websockets" in sys.modules: + response = getattr(exc, "response", None) + status = getattr(response, "status_code", None) + if isinstance(status, int) and type(exc).__module__.startswith("websockets"): + headers = getattr(response, "headers", None) + body = getattr(response, "body", None) + if isinstance(body, bytes): + body = body.decode("utf-8", "replace") + return status, dict(headers or {}), body + status = getattr(exc, "status_code", None) # legacy InvalidStatusCode + if isinstance(status, int) and type(exc).__module__.startswith("websockets"): + return status, dict(getattr(exc, "headers", None) or {}), None + return None + + +def _timeout_types() -> tuple[type[BaseException], ...]: + types: list[type[BaseException]] = [TimeoutError] + if "asyncio" in sys.modules: + import asyncio + + types.append(asyncio.TimeoutError) # a separate class before 3.11 + return tuple(types) + + +def _map_websocket( + exc: BaseException, + secrets: tuple[str, ...], + *, + host: str | None = None, + path: str | None = None, + opening: bool = False, +) -> TransportError | None: + """A WebSocket open that failed: refused upgrade, network or timeout.""" + refused = _ws_status(exc) + if refused is not None: + status, headers, body = refused + return for_status(status, headers, body, secrets, host=host, path=path) + if not opening and "websockets" in sys.modules: + opening = type(exc).__module__.startswith("websockets") + if not opening: + return None + where = _clean(host) if host else _host(exc) + if any(isinstance(item, _timeout_types()) for item in _chain(exc)): + return TransportError( + "timeout", + {"host": where, "timeout_s": _seconds(WS_OPEN_TIMEOUT_S)}, + {"host": where, "timeout_s": _seconds(WS_OPEN_TIMEOUT_S)}, + ) + if any(isinstance(item, OSError) for item in _chain(exc)): + reason = _network_reason(exc) + return TransportError( + "network_unavailable", + {"host": where, "reason": reason}, + {"host": where, "reason": reason}, + ) + if "websockets" in sys.modules: + from websockets.exceptions import InvalidHandshake + + # The server answered the upgrade with something that isn't a valid + # HTTP response (a proxy, HTTP/1.0, a cut-off reply). + if any(isinstance(item, InvalidHandshake) for item in _chain(exc)): + return TransportError( + "network_unavailable", + {"host": where, "reason": REASON_UNREADABLE, "prog": prog() or "dg"}, + {"host": where, "reason": REASON_UNREADABLE}, + NETWORK_UNREADABLE, + ) + return None + + +def map_ws_open( + exc: BaseException, + secrets: tuple[str, ...] = (), + *, + host: str | None = None, + path: str | None = None, +) -> TransportError | None: + """The coded error for a WebSocket open to ``host`` that raised ``exc``. + + A refused upgrade maps by its HTTP status through the REST table; DNS, + connect and TLS failures map to ``network_unavailable``; an open that + timed out maps to ``timeout``. + """ + if isinstance(exc, TransportError): + return exc + return _map_websocket(exc, secrets, host=host, path=path, opening=True) + + def _seconds(value: float) -> int | float: return int(value) if float(value).is_integer() else value +def plain_detail(exc: BaseException) -> str: + """What went wrong, in the words of the error that caused it. + + The client wraps failures it can't map in the SDK's ``ApiError``, + whose text is a dump of its fields (``headers: None, status_code: + None, body: ...``). For one with no status, this is the message of + the error it was raised from; anything else is its own message. + """ + for item in _chain(exc): + if ( + type(item).__name__ == "ApiError" + and getattr(item, "status_code", 1) is None + ): + continue + text = str(item).strip() + if text: + return text + return type(item).__name__ + body = getattr(exc, "body", None) + return str(body) if body else type(exc).__name__ + + +def _running_command() -> str: + """The command being run, as typed (``dg listen``), for messages that + name it; ``dg`` outside a command.""" + if "click" in sys.modules: + import click + + ctx = click.get_current_context(silent=True) + if ctx is not None and ctx.command_path: + return ctx.command_path + return "dg" + + def raise_for( exc: BaseException, secrets: tuple[str, ...] = (), diff --git a/packages/deepctl-core/src/deepctl_core/websocket.py b/packages/deepctl-core/src/deepctl_core/websocket.py index cd4bf4b7..f3f13707 100644 --- a/packages/deepctl-core/src/deepctl_core/websocket.py +++ b/packages/deepctl-core/src/deepctl_core/websocket.py @@ -8,63 +8,25 @@ followed with the same headers (14 and 15, the key included), followed without the key (16 and later, the data still goes), or raised. These helpers turn every 3xx into :class:`RedirectRefused` before a second -connection is opened. REST calls through the SDK don't follow redirects -either (``follow_redirects=False`` in the client); :func:`redirect_refused` -words that error too, with no ``payload``. +connection is opened. REST calls through the shared client don't follow +redirects either; :func:`deepctl_core.transport.for_status` maps their 3xx +to the same :class:`RedirectRefused`. """ from __future__ import annotations from typing import Any -from urllib.parse import urljoin -import click +from .transport import RedirectRefused, redirect_refused -from .hosts import parse_origin -from .messages import display - -E_WS_REDIRECT = ( - "{origin} answered with a redirect to {location_origin}. {command} " - "doesn't follow redirects, so no {payload} was sent." -) -# REST: the request already reached the host that was checked; nothing -# goes to the new location. -E_HTTP_REDIRECT = ( - "{origin} answered with a redirect to {location_origin}. {command} " - "doesn't follow redirects, so nothing was sent there." -) - - -class RedirectRefused(click.ClickException): - """A request was answered with a redirect, which isn't followed.""" - - -def redirect_refused( - url: str, location: str | None, *, command: str, payload: str | None -) -> RedirectRefused: - """The error for a redirect from ``url`` to ``location``. - - Only origins are shown, never paths or queries, and anything a server - put in the Location header is made safe to print first. - """ - here = parse_origin(url) - there = None - if location: - # Header bytes outside ASCII arrive as surrogates; show them as "?". - location = location.encode("ascii", "replace").decode("ascii") - try: - there = parse_origin(urljoin(url, location)) - except ValueError: # e.g. an unbalanced "[" in the host - there = None - text = E_HTTP_REDIRECT if payload is None else E_WS_REDIRECT - return RedirectRefused( - text.format( - origin=display(here.origin if here else "?"), - location_origin=display(there.origin if there else "?"), - command=command, - payload=payload, - ) - ) +# The error and its wording live in transport (one redirect refusal for +# REST and WebSocket); re-exported here for the connect helpers' callers. +__all__ = [ + "RedirectRefused", + "connect_async", + "connect_sync", + "redirect_refused", +] def _as_refusal( diff --git a/packages/deepctl-core/tests/unit/test_auth.py b/packages/deepctl-core/tests/unit/test_auth.py index 229e5823..14862b5a 100644 --- a/packages/deepctl-core/tests/unit/test_auth.py +++ b/packages/deepctl-core/tests/unit/test_auth.py @@ -9,6 +9,7 @@ import pytest from deepctl_core.auth import AuthenticationError, AuthManager from deepctl_core.config import Config +from deepctl_core.transport import request_headers @pytest.fixture @@ -70,7 +71,7 @@ def test_verify_credentials_success(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/test-project", headers={ - "Authorization": "Token sk-test-key", + **request_headers("sk-test-key"), "Content-Type": "application/json", }, ) @@ -160,7 +161,7 @@ def test_verify_credentials_no_project_id(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects", headers={ - "Authorization": "Token sk-test-key", + **request_headers("sk-test-key"), "Content-Type": "application/json", }, ) @@ -205,7 +206,7 @@ def test_verify_credentials_uses_stored_credentials( auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/stored-project", headers={ - "Authorization": "Token sk-stored-key", + **request_headers("sk-stored-key"), "Content-Type": "application/json", }, ) @@ -237,7 +238,7 @@ def test_verify_credentials_uses_env_vars(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/env-project", headers={ - "Authorization": "Token sk-env-key", + **request_headers("sk-env-key"), "Content-Type": "application/json", }, ) diff --git a/packages/deepctl-core/tests/unit/test_auth_insecure.py b/packages/deepctl-core/tests/unit/test_auth_insecure.py index a1e3f810..8a01d8b3 100644 --- a/packages/deepctl-core/tests/unit/test_auth_insecure.py +++ b/packages/deepctl-core/tests/unit/test_auth_insecure.py @@ -173,3 +173,87 @@ def test_other_loopback_addresses_keep_verification(capsys): _warn_if_verified(httpx.Request("GET", "https://127.0.0.2/")) assert "applies only to localhost" in capsys.readouterr().err + + +# -- The shared API client (SDK) and dg api's verified client -------------- + + +@pytest.fixture +def fresh_transport(monkeypatch): + from deepctl_core import transport + + monkeypatch.setattr(transport, "_client", None) + monkeypatch.setattr(transport, "_client_insecure", False) + monkeypatch.setattr(transport, "_verified", None) + return transport + + +def _verifies(client: httpx.Client, url: str) -> bool: + import ssl + + pool = client._transport_for_url(httpx.URL(url))._pool # type: ignore[attr-defined] + return pool._ssl_context.verify_mode == ssl.CERT_REQUIRED + + +@pytest.mark.parametrize("value", ["1", "true", "TRUE"]) +def test_shared_client_skips_verification_for_loopback_only(fresh_transport, monkeypatch, value): + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", value) + shared = fresh_transport.shared_client() + assert not _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert not _verifies(shared, "https://localhost:8443/v1/models") + assert not _verifies(shared, "https://[::1]:8443/v1/models") + for url in ( + "https://api.deepgram.com/v1/models", + "https://mitm.example:8443/v1/models", + "https://127.0.0.2:8443/v1/models", + ): + assert _verifies(shared, url), url + # dg api's client verifies every host, loopback included (as in 0.4.1). + verified = fresh_transport.verified_client() + assert verified is not shared + assert _verifies(verified, "https://127.0.0.1:8443/v1/projects") + assert verified.follow_redirects is False + + +@pytest.mark.parametrize("value", ["0", "false", "no", ""]) +def test_shared_client_verifies_for_other_values(fresh_transport, monkeypatch, value): + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", value) + shared = fresh_transport.shared_client() + assert _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert _verifies(shared, "https://mitm.example:8443/v1/models") + assert fresh_transport.verified_client() is shared # one client per process + + +def test_shared_client_verifies_for_agents(fresh_transport, monkeypatch, capsys): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + shared = fresh_transport.shared_client() + assert _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert "Ignoring DEEPGRAM_CLI_INSECURE" in capsys.readouterr().err + + +def test_shared_client_loopback_goes_direct(fresh_transport, proxy, capsys): + server = _Recorder() + try: + try: + fresh_transport.shared_client().get( + f"https://127.0.0.1:{server.port}/v1/models", timeout=3 + ) + except httpx.HTTPError: + pass + finally: + server.close() + assert proxy.lines == [] + assert len(server.lines) == 1 # TLS ClientHello, straight to the server + + +def test_shared_client_remote_host_keeps_proxy_and_warns(fresh_transport, proxy, capsys): + for _ in range(2): + try: + fresh_transport.shared_client().get( + "https://api.deepgram.example/v1/models", timeout=3 + ) + except httpx.HTTPError: + pass + assert proxy.lines == ["CONNECT api.deepgram.example:443 HTTP/1.1"] * 2 + err = capsys.readouterr().err + assert err.count("DEEPGRAM_CLI_INSECURE applies only to localhost") == 1 diff --git a/packages/deepctl-core/tests/unit/test_client.py b/packages/deepctl-core/tests/unit/test_client.py index c6bfeb68..ac8b104a 100644 --- a/packages/deepctl-core/tests/unit/test_client.py +++ b/packages/deepctl-core/tests/unit/test_client.py @@ -72,10 +72,13 @@ def test_client_property_creates_client( # Verify auth guard was called mock_auth_manager.guard.assert_called_once() - # Verify DGClient was created with keyword args, without redirects + # Verify DGClient was created with keyword args, on the shared + # client, which follows no redirects assert mock_dg_client.called assert mock_dg_client.call_args.kwargs["api_key"] == "sk-test" - assert mock_dg_client.call_args.kwargs["follow_redirects"] is False + http = mock_dg_client.call_args.kwargs["httpx_client"] + assert http is transport.shared_client() + assert http.follow_redirects is False # Verify client is cached assert client._client == mock_instance @@ -130,8 +133,8 @@ def test_create_client_with_custom_base_url( mock_dg_client.assert_called_once_with( api_key="sk-test", httpx_client=transport.shared_client(), + session_id=transport.session_id(), environment=mock_env.return_value, - follow_redirects=False, ) @patch("deepctl_core.client.DGClient") diff --git a/packages/deepctl-core/tests/unit/test_transport.py b/packages/deepctl-core/tests/unit/test_transport.py index 3b980e6c..03fb23f5 100644 --- a/packages/deepctl-core/tests/unit/test_transport.py +++ b/packages/deepctl-core/tests/unit/test_transport.py @@ -74,7 +74,7 @@ def record(request: httpx.Request) -> httpx.Response: seen.append(request) return handler(request) - # Same redirect setting as transport.shared_client(). + # _set_client applies shared_client()'s settings (no redirects, no cookies). transport._set_client( httpx.Client(transport=httpx.MockTransport(record), follow_redirects=False) ) @@ -228,11 +228,112 @@ def test_network_rows(factory, reason): assert err.error == ( "The connection to api.deepgram.com dropped before the response finished." ) + elif reason in ("connection reset", "connection closed"): + # Post-connect: the request had started. + assert err.messages == transport.NETWORK_DROPPED + assert err.error == ( + "The connection to api.deepgram.com dropped before the response " + f"was complete ({reason})." + ) else: assert err.messages == transport.CODES["network_unavailable"] assert err.error == f"Couldn't reach api.deepgram.com: {reason}." +@pytest.mark.parametrize( + ("factory", "reason", "messages"), + [ + # A reset or close while connecting is still "couldn't reach". + ( + _chained( + httpx.ConnectError, ConnectionResetError(54, "Connection reset by peer") + ), + "connection reset", + "E-NETWORK", + ), + ( + lambda r: httpx.ConnectError("Connection reset by peer", request=r), + "connection reset", + "E-NETWORK", + ), + # TLS failing on a read is a TLS failure, not a drop. + ( + _chained(httpx.ReadError, ssl.SSLError(1, "[SSL] record layer failure")), + "TLS handshake failed", + "E-NETWORK", + ), + # After connect: the upload was being written, or the response read. + ( + _chained(httpx.WriteError, BrokenPipeError(32, "Broken pipe")), + "connection reset", + "E-NETWORK-DROPPED", + ), + ( + _chained( + httpx.WriteError, ConnectionResetError(104, "Connection reset by peer") + ), + "connection reset", + "E-NETWORK-DROPPED", + ), + ( + lambda r: httpx.ReadError("", request=r), + "connection reset", + "E-NETWORK-DROPPED", + ), + ], +) +def test_dropped_only_after_connect(factory, reason, messages): + err = mapped(raising(factory)) + assert err.code == "network_unavailable" + assert err.details == {"host": "api.deepgram.com", "reason": reason} + assert err.messages[0] == messages + + +def _reset_after_full_body() -> tuple[int, threading.Thread]: + """A loopback server that reads the whole request, then resets.""" + import struct + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + + def serve() -> None: + conn, _ = listener.accept() + data = b"" + while b"\r\n\r\n" not in data: + data += conn.recv(65536) + head, _, body = data.partition(b"\r\n\r\n") + length = next( + int(line.split(b":", 1)[1]) + for line in head.split(b"\r\n") + if line.lower().startswith(b"content-length:") + ) + while len(body) < length: + body += conn.recv(65536) + conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + conn.close() + listener.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return listener.getsockname()[1], thread + + +def test_reset_after_the_full_body_is_dropped_not_unreachable(): + """The whole upload arrived, then the server reset. That isn't + "couldn't reach": the request had started (E-NETWORK-DROPPED).""" + port, thread = _reset_after_full_body() + with httpx.Client() as client, pytest.raises(httpx.TransportError) as raised: + client.post(f"http://127.0.0.1:{port}/v1/listen", content=b"x" * 3145729) + thread.join(5) + err = transport.map_exception(raised.value) + assert err is not None and err.code == "network_unavailable" + # macOS raises ReadError (reset); Linux may see EOF (closed) instead. + assert err.details["reason"] in ("connection reset", "connection closed") + assert err.messages == transport.NETWORK_DROPPED + assert err.error.startswith("The connection to 127.0.0.1 dropped before") + + def test_unreadable_2xx_hint_names_host_and_prog(monkeypatch): err = mapped(status(200, b"sign in")) payload = _render(err, monkeypatch, agentic=False, hard=False) @@ -308,14 +409,70 @@ def broken(*args, **kwargs): def test_every_code_is_catalogued(): for code, ids in [ *transport.CODES.items(), + ("dropped", transport.NETWORK_DROPPED), ("cut", transport.NETWORK_CUT), ("unreadable", transport.NETWORK_UNREADABLE), + ("redirect", transport.NETWORK_REDIRECT), ]: for message_id in ids: if message_id: assert message_id in transport.STRINGS, (code, message_id) +# What dg prints for a refused redirect: 0.4.1's text, origins only. +_REDIRECT_CASES = [ + ( + {"command": "dg listen"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg listen doesn't follow redirects, so nothing " + "was sent there.", + ), + ( + {"command": "dg listen", "payload": "audio"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg listen doesn't follow redirects, so no audio " + "was sent.", + ), + ( + {"command": "dg speak", "payload": "text"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg speak doesn't follow redirects, so no text " + "was sent.", + ), +] + + +def _refused(**kwargs: Any) -> RedirectRefused: + return transport.redirect_refused( + "https://api.deepgram.com/v1/listen?model=nova-3", + "https://evil.example/steal?k=1", + **kwargs, + ) + + +@pytest.mark.parametrize(("kwargs", "printed"), _REDIRECT_CASES) +def test_redirect_refusal_prints_e_network_redirect(kwargs, printed): + """The printed refusal is E-NETWORK-REDIRECT filled in, so the catalogue + entry that holds that string is the text users see.""" + err = _refused(**kwargs) + assert err.message == err.error == printed + assert err.messages[0] == "E-NETWORK-REDIRECT" + assert transport.STRINGS["E-NETWORK-REDIRECT"].format(**err.values) == printed + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +@pytest.mark.parametrize(("kwargs", "printed"), _REDIRECT_CASES) +def test_catalogue_redirect_text_is_what_dg_prints(kwargs, printed): + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + template = catalogue["strings"]["E-NETWORK-REDIRECT"] + for name in ("origin", "location_origin", "command", "sent"): + assert "{%s}" % name in catalogue["placeholders"], name + assert template.format(**_refused(**kwargs).values) == printed + + @pytest.mark.skipif( not os.environ.get("DG_ERROR_CODES_JSON"), reason="DG_ERROR_CODES_JSON names no error catalogue file", @@ -343,13 +500,26 @@ def test_strings_match_catalogue(): assert message.get("hint_automation") == automation entry = catalogue["codes"]["network_unavailable"] for extra, reason in ( + (transport.NETWORK_DROPPED, transport.REASON_RESET), + (transport.NETWORK_DROPPED, transport.REASON_CLOSED), (transport.NETWORK_CUT, "response cut off"), (transport.NETWORK_UNREADABLE, "unreadable response"), + (transport.NETWORK_REDIRECT, transport.REASON_REDIRECT), ): message = next(m for m in entry["messages"] if m["id"] == extra[0]) assert (message["hint_human"], message["hint_agent"]) == extra[1:3] assert "hint_automation" not in message - assert entry["details_values"][extra[0]] == {"reason": [reason]} + assert reason in entry["details_values"][extra[0]]["reason"] + # A 3xx with credentials isn't followed, and has its own message. + assert transport.REASON_REDIRECT not in ( + entry["details_values"]["E-NETWORK-UNREADABLE"]["reason"] + ) + assert catalogue["transport_mapping"][0][0].split("|")[-1] == "3xx" + for message_id in ("W-ENV-PLACEHOLDER", "W-KEYRING-TIMEOUT"): + from deepctl_core import credentials + + text = getattr(credentials, message_id.replace("-", "_")) + assert catalogue["strings"][message_id] == text env = catalogue["codes"]["auth_invalid"]["env_key_variant"] assert env == { "hint_agent": "A-AUTH-INVALID-ENV", @@ -734,10 +904,11 @@ def handler(request): return httpx.Response(200, json={"models": []}) client, _ = facade(handler) - with pytest.raises(RedirectRefused): + with pytest.raises(TransportError) as err: client.list_models() assert [r.url.host for r in seen] == ["api.deepgram.com"] assert seen[0].headers["Authorization"] == f"Token {KEY}" + assert err.value.details["reason"] == "unexpected redirect" def test_shared_client_is_lazy_and_single(): @@ -758,3 +929,321 @@ def test_print_warning_ignore_quiet(ignore_quiet, monkeypatch, capsys): monkeypatch.setitem(output._output_config, "agentic", True) output.print_warning("needed", stderr=True, ignore_quiet=ignore_quiet) assert ("WARN: needed" in capsys.readouterr().err) is ignore_quiet + + +# ── One Transport ──────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("base", "rest", "ws"), + [ + ( + "https://api.deepgram.com", + "https://api.deepgram.com", + "wss://api.deepgram.com", + ), + ("http://127.0.0.1:8080", "http://127.0.0.1:8080", "ws://127.0.0.1:8080"), + ("ws://127.0.0.1:8080/x", "http://127.0.0.1:8080/x", "ws://127.0.0.1:8080/x"), + ("wss://h.example:9", "https://h.example:9", "wss://h.example:9"), + ("h.example", "https://h.example", "wss://h.example"), + ], +) +def test_base_derivation(base, rest, ws): + assert transport.rest_url(base) == rest + assert transport.ws_url(base) == ws + assert transport.ws_url(transport.rest_url(base)) == ws + + +def test_transport_bases_come_from_the_host_check(monkeypatch): + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080/") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + assert t.rest_base() == "http://127.0.0.1:8080" + assert t.ws_base() == "ws://127.0.0.1:8080" + assert t.auth_headers(KEY) == transport.request_headers(KEY) + assert t.auth_headers(KEY)["Authorization"] == f"Token {KEY}" + assert t.client is transport.shared_client() + + +def test_request_headers_are_the_sdk_clients_headers(monkeypatch): + """dg's own requests and the SDK's carry the same User-Agent, X-Fern + headers, session ID and Authorization: one run, one session ID.""" + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080/") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + sdk = t.sdk_client(KEY) + assert sdk._client_wrapper.get_headers() == t.auth_headers(KEY) + sid = t.auth_headers(KEY)[transport.SESSION_HEADER] + assert sid == transport.session_id() == sdk.session_id + assert len(sid) == 36 + assert t.auth_headers("sk-other")[transport.SESSION_HEADER] == sid + + +def test_no_header_for_a_refused_host(monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + with pytest.raises(hosts.CredentialHostError): + t.auth_headers(KEY) + with pytest.raises(hosts.CredentialHostError): + t.sdk_client(KEY) + + +def test_default_host_is_hosts_constant(): + assert transport.default_host() == "api.deepgram.com" + assert f"https://{transport.default_host()}" == hosts.DEFAULT_BASE_URL + + +def test_backend_seam(monkeypatch): + calls = [] + + class Fake: + name = "fake" + + def sdk_client(self, api_key, rest_base, ws_base): + calls.append((api_key, rest_base, ws_base)) + return "sdk" + + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080") + config = Config(config_path=str(Path.cwd() / "none.yaml")) + client = DeepgramClient(config, AuthManager(config)) + assert isinstance(client.transport.backend, transport.RestBackend) + client.transport.backend = Fake() + assert client.client == "sdk" + assert calls == [(KEY, "http://127.0.0.1:8080", "ws://127.0.0.1:8080")] + + +@pytest.mark.parametrize("code", [301, 302, 303, 307, 308]) +@pytest.mark.parametrize( + "call", + [ + lambda c: c.list_models(), + lambda c: c.get_projects(), + ], +) +def test_redirects_are_not_followed(code, call): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + code, headers={"location": "https://sink.example/v1/projects"} + ) + + client, seen = facade(handler) + with pytest.raises(TransportError) as err: + call(client) + assert [r.url.host for r in seen] == ["api.deepgram.com"] + assert err.value.code == "network_unavailable" + assert err.value.messages == transport.NETWORK_REDIRECT + assert err.value.details == { + "host": "api.deepgram.com", + "reason": "unexpected redirect", + } + assert err.value.error == ( + "https://api.deepgram.com answered with a redirect to " + "https://sink.example. dg doesn't follow redirects, so nothing was " + "sent there." + ) + + +@pytest.mark.parametrize( + ("agentic", "hard", "hint"), + [ + ( + False, + False, + "A network sign-in page or proxy may be redirecting requests. If " + "api.deepgram.com is your own server, use the address it redirects to.", + ), + ( + True, + True, + "Tell the user api.deepgram.com redirected the request. Don't retry " + "until they have checked the network or the configured URL.", + ), + ], +) +def test_redirect_hints(monkeypatch, agentic, hard, hint): + err = mapped(status(302, headers={"location": "https://sink.example/"})) + payload = _render(err, monkeypatch, agentic=agentic, hard=hard) + assert payload["hint"] == hint + assert payload["next_step"] is None + + +def test_redirect_on_upload_sends_the_body_once(tmp_path): + audio = tmp_path / "a.wav" + audio.write_bytes(b"RIFF" + b"\0" * 64) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(307, headers={"location": "https://sink.example/x"}) + + client, seen = facade(handler) + with pytest.raises(TransportError) as err: + client.transcribe_file(audio) + assert len(seen) == 1 and seen[0].url.path == "/v1/listen" + assert err.value.details["reason"] == "unexpected redirect" + + +def test_shared_client_follows_no_redirects(monkeypatch): + created = {} + + class Recorder(httpx.Client): + def __init__(self, **kwargs): + created.update(kwargs) + super().__init__(**kwargs) + + monkeypatch.setattr(httpx, "Client", Recorder) + monkeypatch.delenv("DEEPGRAM_CLI_INSECURE", raising=False) + client = transport.shared_client() + assert created["follow_redirects"] is False + assert created.get("verify", True) is True + assert "mounts" not in created + assert client.follow_redirects is False + + +@pytest.mark.parametrize(("agent", "loopback_mounts"), [(False, True), (True, False)]) +def test_insecure_flag_only_for_a_person(monkeypatch, capsys, agent, loopback_mounts): + created = {} + + class Recorder(httpx.Client): + def __init__(self, **kwargs): + created.update(kwargs) + super().__init__(**kwargs) + + monkeypatch.setattr(httpx, "Client", Recorder) + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", "1") + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + transport.shared_client() + # Verification stays on; a person gets unverified loopback mounts only + # (test_auth_insecure.py checks hosts and dg api's verified client). + assert created.get("verify", True) is True + assert ("mounts" in created) is loopback_mounts + err = capsys.readouterr().err + assert ("Ignoring DEEPGRAM_CLI_INSECURE" in err) is agent + + +def test_cookies_are_never_sent_back(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"models": []} if "models" in request.url.path else {"projects": []}, + headers={"set-cookie": "sid=s3cr3t; Path=/; Domain=api.deepgram.com"}, + ) + + client, seen = facade(handler) + client.list_models() + client.get_projects() + client.list_models() + assert len(seen) == 3 + assert all("cookie" not in r.headers for r in seen) + assert len(transport.shared_client().cookies.jar) == 0 + + +def test_no_code_sets_cookies_on_the_shared_client(): + """httpx copies the client's jar into each request, so a + cookie set in code would be sent anyway. Nothing may set one.""" + import re + + root = Path(__file__).resolve().parents[3] # packages/ + offenders = [] + files = list(root.glob("*/src/**/*.py")) + assert len(files) > 50 + for path in files: + text = path.read_text(encoding="utf-8") + for number, line in enumerate(text.splitlines(), 1): + if re.search(r"\.cookies\.set\(|\bcookies\s*=", line): + offenders.append(f"{path.relative_to(root)}:{number}: {line.strip()}") + assert offenders == [] + + +# ── WebSocket handshake mapping ────────────────────────────────────── + + +def _ws_refused(code: int, headers: dict | None = None) -> BaseException: + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatus + from websockets.http11 import Response + + return InvalidStatus(Response(code, "x", Headers(headers or {}), b"{}")) + + +@pytest.mark.parametrize( + ("code", "want"), + [ + (401, "auth_invalid"), + (403, "auth_forbidden"), + (429, "rate_limited"), + (400, "invalid_argument"), + (500, "server_error"), + (503, "server_error"), + (302, "network_unavailable"), + ], +) +def test_ws_refused_upgrade_maps_by_status(code, want): + exc = _ws_refused(code, {"retry-after": "7", "dg-request-id": "r-1"}) + mapped = transport.map_exception(exc) + assert mapped is not None and mapped.code == want + assert transport.map_ws_open(exc, host="api.deepgram.com").code == want + if code == 429: + assert mapped.details["retry_after"] == 7 + if code == 302: + assert mapped.details["reason"] == "unexpected redirect" + assert mapped.messages == transport.NETWORK_REDIRECT + + +def test_ws_legacy_invalid_status_code(): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatusCode + + exc = InvalidStatusCode(401, Headers()) + assert transport.map_exception(exc).code == "auth_invalid" + + +@pytest.mark.parametrize( + ("inner", "code", "reason"), + [ + ( + socket.gaierror(-2, "Name or service not known"), + "network_unavailable", + "name resolution failed", + ), + ( + ConnectionRefusedError(111, "refused"), + "network_unavailable", + "connection refused", + ), + ( + ssl.SSLError(1, "CERTIFICATE_VERIFY_FAILED"), + "network_unavailable", + "TLS handshake failed", + ), + (TimeoutError(), "timeout", None), + ], +) +def test_ws_open_failures(inner, code, reason): + try: + try: + raise inner + except BaseException as cause: + raise RuntimeError("open failed") from cause + except RuntimeError as exc: + mapped = transport.map_ws_open(exc, host="api.deepgram.com") + assert mapped.code == code + assert mapped.details["host"] == "api.deepgram.com" + if reason: + assert mapped.details["reason"] == reason + else: + assert mapped.details["timeout_s"] == transport._seconds( + transport.WS_OPEN_TIMEOUT_S + ) + + +def test_unrelated_errors_are_not_ws_errors(): + assert transport.map_exception(ValueError("x")) is None + assert transport.map_ws_open(ValueError("x"), host="h") is None + + +def test_module_aliases_kept_for_w24_w25(): + assert callable(transport.shared_client) + assert callable(transport.map_exception) + assert callable(transport.raise_for) diff --git a/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py b/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py new file mode 100644 index 00000000..17dd21bf --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py @@ -0,0 +1,104 @@ +"""A WebSocket upgrade answered with something that isn't valid HTTP, and +errors the client wraps for the SDK: plain words, no field dumps.""" + +from __future__ import annotations + +import pytest +from deepctl_core import transport + +websockets_exceptions = pytest.importorskip("websockets.exceptions") + + +def _wrapped(cause: BaseException) -> BaseException: + from deepgram.core.api_error import ApiError + + try: + try: + raise cause + except Exception as e: + raise ApiError(body=f"Text-to-speech streaming failed: {e}") from None + except ApiError as wrapped: + wrapped.__context__ = cause + return wrapped + + +def test_invalid_http_answer_maps_to_unreadable(): + exc = websockets_exceptions.InvalidMessage( + "did not receive a valid HTTP response" + ) + exc.__cause__ = EOFError("connection closed while reading HTTP status line") + mapped = transport.map_exception(exc) + assert isinstance(mapped, transport.TransportError) + assert mapped.code == "network_unavailable" + assert mapped.details["reason"] == transport.REASON_UNREADABLE + + +def test_invalid_header_maps_too(): + mapped = transport.map_ws_open( + websockets_exceptions.InvalidHeader("Upgrade"), host="api.deepgram.com" + ) + assert mapped is not None + assert mapped.code == "network_unavailable" + assert mapped.details == { + "host": "api.deepgram.com", + "reason": transport.REASON_UNREADABLE, + } + + +def test_refused_status_still_maps_by_status(): + from websockets.datastructures import Headers + from websockets.http11 import Response + + response = Response(401, "Unauthorized", Headers(), b"") + mapped = transport.map_exception(websockets_exceptions.InvalidStatus(response)) + assert mapped is not None + assert mapped.code == "auth_invalid" + + +def test_plain_detail_skips_the_sdk_field_dump(): + wrapped = _wrapped(RuntimeError("socket closed")) + assert "status_code" in str(wrapped) # the dump this avoids + assert transport.plain_detail(wrapped) == "socket closed" + + +def test_plain_detail_keeps_an_ordinary_message(): + assert transport.plain_detail(ValueError("bad value")) == "bad value" + assert transport.plain_detail(ValueError()) == "ValueError" + + +def test_plain_detail_keeps_an_api_error_with_a_status(): + from deepgram.core.api_error import ApiError + + err = ApiError(status_code=500, body="oops") + assert "500" in transport.plain_detail(err) + + +def test_rest_redirect_names_the_running_command(): + import click + + @click.group(name="dg") + def root() -> None: + pass + + @root.command(name="models") + def models() -> None: + err = transport.for_status( + 307, {"location": "https://elsewhere.example/x"}, host="api.deepgram.com" + ) + assert err is not None + click.echo(err.message) + + from click.testing import CliRunner + + result = CliRunner().invoke(root, ["models"]) + assert "dg models doesn't follow redirects, so nothing was sent there." in ( + " ".join(result.output.split()) + ), result.output + + +def test_rest_redirect_outside_a_command_says_dg(): + err = transport.for_status( + 307, {"location": "https://elsewhere.example/x"}, host="api.deepgram.com" + ) + assert err is not None + assert "dg doesn't follow redirects" in err.message From 638a3459c2e9095c392111ebbf6d731a24538772 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 18:55:55 -0400 Subject: [PATCH 3/8] fix(debug): take the probe's and network's host from the transport debug probe's upstream is the Transport's checked WebSocket base instead of a hard-coded api.deepgram.com, so it follows the profile's base URL and a refused host never gets the key. The Authorization header comes from the Transport after its host check. User-visible changes: - An upstream that rejects the key at the upgrade (401) stops the proxy with the coded error, exit 1, because every connection would fail the same way. Other open streams are closed first, so it exits within seconds. - Any other refused upgrade (including 403, which is about one request's scope), an unreachable upstream, or an open that takes over 10 s fails only that client's connection, with the coded error on stderr; the proxy keeps serving. - The proxy's status lines go to stderr, so stdout holds only the result. debug network's default endpoint, its latency row and its proxy advice name the host from hosts.DEFAULT_BASE_URL, through the Transport. It still sends no credentials. debug toolkit's examples build their streaming URL from the same constant. The one api.deepgram.com left in a .py source outside hosts.py is the foreign-host hint in messages.py, which a person copies; it's marked 'host-literal: user text'. --- .../src/deepctl_cmd_debug_network/command.py | 22 +- .../tests/unit/test_network_command.py | 14 ++ .../src/deepctl_cmd_debug_probe/command.py | 36 ++- .../src/deepctl_cmd_debug_probe/proxy.py | 106 +++++--- .../tests/unit/test_probe.py | 235 +++++++++++++++++- .../tests/unit/test_proxy_loopback.py | 142 ++++++++--- .../src/deepctl_cmd_debug_toolkit/command.py | 10 +- .../deepctl-core/src/deepctl_core/messages.py | 2 +- 8 files changed, 484 insertions(+), 83 deletions(-) diff --git a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py index d80b0fb8..fab19427 100644 --- a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py +++ b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py @@ -32,11 +32,15 @@ console = Console() -# Default Deepgram endpoints to test -DEEPGRAM_ENDPOINTS = { - "api": "https://api.deepgram.com", - "auth": "https://auth.dx.deepgram.com", -} + +def _default_host() -> str: + """The Deepgram API host (``hosts.DEFAULT_BASE_URL``, via the Transport). + + The default endpoint only: this command sends no credentials anywhere. + """ + from deepctl_core import transport + + return transport.default_host() def _is_ssl_error(exc: BaseException) -> bool: @@ -131,7 +135,7 @@ def get_arguments(self) -> list[dict[str, Any]]: return [ { "names": ["--endpoint", "-e"], - "help": ("Specific endpoint to test (default: api.deepgram.com)"), + "help": f"Specific endpoint to test (default: {_default_host()})", "type": str, "is_option": True, }, @@ -205,7 +209,7 @@ def handle( **kwargs: Any, ) -> Any: """Handle network debug command execution.""" - endpoint = kwargs.get("endpoint") or "api.deepgram.com" + endpoint = kwargs.get("endpoint") or _default_host() verbose = kwargs.get("verbose", False) skip_commands = kwargs.get("skip_commands", False) timeout = kwargs.get("timeout", 10) @@ -744,7 +748,7 @@ def _test_multi_region_latency( ) -> None: """Test latency to known Deepgram endpoints.""" endpoints = [ - ("api.deepgram.com", "Primary API"), + (_default_host(), "Primary API"), ] console.print("\n[bold]Multi-Region Latency:[/bold]") @@ -1054,7 +1058,7 @@ def _generate_recommendations(self, result: NetworkDebugResult) -> None: recommendations.append( "ℹ️ Proxy configuration detected. Ensure your proxy allows access to:" ) - recommendations.append(" • api.deepgram.com (port 443)") + recommendations.append(f" • {_default_host()} (port 443)") recommendations.append(" • Certificate validation endpoints (port 80)") # Display recommendations diff --git a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py index 667a17dd..e795bd60 100644 --- a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py +++ b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py @@ -707,3 +707,17 @@ def test_missing_bundle_is_reported_like_requests( assert with_verify.success is False assert "Could not find a suitable TLS CA certificate bundle" in with_verify.error assert without_verify.success is True + + +def test_default_endpoint_comes_from_hosts_default(): + """The default host is hosts.DEFAULT_BASE_URL's, through the + Transport; debug network sends no credentials anywhere.""" + from pathlib import Path + + from deepctl_cmd_debug_network import command as network_module + from deepctl_core.hosts import DEFAULT_BASE_URL + + assert f"https://{network_module._default_host()}" == DEFAULT_BASE_URL + source = Path(network_module.__file__).read_text(encoding="utf-8") + for credential_use in ("get_api_key", "Authorization", "auth_headers"): + assert credential_use not in source diff --git a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py index fd67c637..8a4c08eb 100644 --- a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py +++ b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py @@ -6,14 +6,20 @@ import time from typing import Any -from deepctl_core import AuthManager, BaseCommand, Config, DeepgramClient +from deepctl_core import ( + AuthManager, + BaseCommand, + Config, + DeepgramClient, + get_status_console, +) from deepctl_shared_utils import require_ffprobe -from rich.console import Console from rich.panel import Panel from .models import ProbeDebugResult -console = Console() +# Status chrome is diagnostics: stderr, so stdout carries only the result. +console = get_status_console() def _find_available_port(start: int = 3100, end: int = 3199) -> int | None: @@ -165,24 +171,33 @@ def handle( port=port, ) - upstream_host = "api.deepgram.com" + from deepctl_core import transport + from rich.markup import escape + + # The upstream is the Transport's checked WebSocket base, and the key + # goes only where that check allows (refused hosts raise, reported). + upstream = transport.Transport(config, check_host=auth_manager.check_base_url) + upstream_base = upstream.ws_base() + upstream_headers = upstream.auth_headers(api_key) # Create proxy proxy = ProbeProxy( api_key=api_key, config=config, - upstream_host=upstream_host, + upstream_base=upstream_base, probe_interval_bytes=probe_interval_bytes, probe_interval_seconds=probe_interval_seconds, verbose=verbose, + upstream_headers=upstream_headers, ) + upstream_host = proxy.upstream_host # Print instructions console.print( Panel( f"[bold cyan]WebSocket Probe Proxy[/bold cyan]\n\n" f"Listening on: [green]ws://localhost:{port}[/green]\n" - f"Upstream: [yellow]wss://{upstream_host}[/yellow]\n" + f"Forwards to: [yellow]{escape(proxy.upstream_base)}[/yellow]\n" f"API key: [dim]****{api_key[-4:]}[/dim]\n\n" f"Probe every: {probe_interval_bytes:,} bytes " f"or {probe_interval_seconds}s\n\n" @@ -205,7 +220,10 @@ def interrupt() -> None: if proxy._stop.is_set(): proxy.abort() else: - console.print("\n[yellow]Shutting down probe proxy...[/yellow]") + console.print( + "\n[yellow]Stopping: open connections can finish. Press " + "Ctrl+C again to end them now.[/yellow]" + ) proxy._stop.set() installed, previous = _handle_sigint(loop, interrupt) @@ -231,6 +249,10 @@ def interrupt() -> None: signal.signal(signal.SIGINT, previous) loop.close() + if proxy.fatal is not None: + # The key was rejected at the upgrade: exit 1 with its code. + raise proxy.fatal + duration = time.time() - start_time console.print(f"\n[green]Probe proxy stopped after {duration:.1f}s[/green]") diff --git a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py index ba2834cd..3a90cad4 100644 --- a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py +++ b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py @@ -14,12 +14,12 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit +from deepctl_core.output import get_status_console from deepctl_core.websocket import RedirectRefused from deepctl_shared_utils.ffprobe import ( _parse_probe_data, # lockstep sibling; same parsing as probe_buffer get_ffprobe_path, ) -from rich.console import Console from rich.text import Text from websockets.asyncio.server import Server, ServerConnection, serve from websockets.exceptions import ConnectionClosed, WebSocketException @@ -29,11 +29,18 @@ if TYPE_CHECKING: from deepctl_core import Config + from deepctl_core.transport import TransportError from deepctl_shared_utils.ffprobe_models import AudioProbeResult from websockets.asyncio.client import ClientConnection from websockets.http11 import Request, Response -console = Console() +# Status and transcripts are diagnostics: stderr, so stdout carries only the +# command's result (one JSON document under -o json). +console = get_status_console() + +# Refusals that end the proxy: the key is wrong for every connection. A 403 +# is about this request's scope (another path may be allowed), so it isn't. +_FATAL_CODES = frozenset({"auth_invalid"}) # Frames up to 4 MiB each way, the limit earlier releases allowed. MAX_FRAME_BYTES = 4 * 2**20 @@ -71,21 +78,33 @@ def __init__( self, api_key: str, config: Config | None = None, - upstream_host: str = "api.deepgram.com", + upstream_base: str | None = None, probe_interval_bytes: int = 131072, probe_interval_seconds: float = 10.0, verbose: bool = False, + upstream_headers: dict[str, str] | None = None, ) -> None: + """``upstream_base`` and ``upstream_headers`` come from the command's + Transport: the checked WebSocket base and the ``Authorization`` it + attaches only after the host check passed. Without headers, the + upstream gets no key.""" + from deepctl_core import transport + from deepctl_core.hosts import DEFAULT_BASE_URL + self.api_key = api_key self.config = config - self.upstream_host = upstream_host + # Where connections go: the base, then the client's path and query. + self.upstream_base = transport.ws_url(upstream_base or DEFAULT_BASE_URL) + self.upstream_base = self.upstream_base.rstrip("/") + self.upstream_host = urlsplit(self.upstream_base).hostname or "" + self.upstream_headers = dict(upstream_headers or {}) self.probe_interval_bytes = probe_interval_bytes self.probe_interval_seconds = probe_interval_seconds self.verbose = verbose self.connections: list[ProbeConnectionStats] = [] - # Where connections go: wss:// plus the fixed host, then the client's - # path and query. Tests point it at a loopback ws:// server. - self._upstream_base = f"wss://{upstream_host}" + # A coded error from an upstream that refused the key at the upgrade: + # the proxy stops and the command reports it (auth_invalid). + self.fatal: TransportError | None = None self._stop = asyncio.Event() # Open client and upstream connections, and pending upstream # connects, so a second Ctrl+C can end them at once. @@ -139,9 +158,10 @@ def process_request( async def _connect_upstream( self, url: str, headers: dict[str, str] ) -> ClientConnection: - """Open the upstream WebSocket. The API key goes only here, and a - redirect on the upgrade is refused (``RedirectRefused``), never - followed.""" + """Open the upstream WebSocket, within the Transport's open timeout. + The API key goes only here, and a redirect on the upgrade is refused + (``RedirectRefused``, a coded network_unavailable), never followed.""" + from deepctl_core import transport from deepctl_core.websocket import open_async conn: ClientConnection = await open_async( @@ -149,7 +169,7 @@ async def _connect_upstream( headers, command="dg debug probe", payload="audio", - open_timeout=10, + open_timeout=transport.WS_OPEN_TIMEOUT_S, close_timeout=30, max_size=MAX_FRAME_BYTES, compression=None, @@ -188,15 +208,34 @@ def abort(self) -> None: for proc in list(self._ffprobes): self._reaping.add(asyncio.ensure_future(self._stop_ffprobe(proc))) + async def close_all(self) -> None: + """Close every open connection on both sides (each within 2 s).""" + for conn in list(self._live): + with contextlib.suppress(Exception): + await asyncio.wait_for(conn.close(), 2) + + async def _stop_all(self) -> None: + """A fatal stop: close every open stream, then stop serving.""" + await self.close_all() + self._stop.set() + async def run(self, port: int) -> None: """Serve until `_stop` is set or this task is cancelled; then close - the listener and wait for every connection's summary.""" + the listener and wait for every connection's summary. After a fatal + refusal that wait is capped at 2 s; what's left is aborted.""" server = await self.start(port) try: await self._stop.wait() finally: server.close() - await server.wait_closed() + if self.fatal is not None: + closed = asyncio.ensure_future(server.wait_closed()) + done, _ = await asyncio.wait({closed}, timeout=2) + if not done: + self.abort() + await closed + else: + await server.wait_closed() await asyncio.gather(*self._reaping) async def handle_connection(self, ws_client: ServerConnection) -> None: @@ -213,11 +252,11 @@ async def handle_connection(self, ws_client: ServerConnection) -> None: ) self.connections.append(stats) - upstream_url = self._upstream_base + target + upstream_url = self.upstream_base + target - upstream_headers = { - "Authorization": f"Token {self.api_key}", - } + # The Transport's Authorization, attached after its host check; the + # proxy never builds it. + upstream_headers = dict(self.upstream_headers) console.print( Text.from_markup( @@ -241,21 +280,32 @@ async def handle_connection(self, ws_client: ServerConnection) -> None: self._connecting.add(connecting) try: ws_upstream = await connecting - except RedirectRefused as e: - console.print( - Text.from_markup( - f"[red]Upstream connection refused[/red] [{conn_id}]: " - ), - untrusted(e.message), - sep="", + except ( + RedirectRefused, + WebSocketException, + OSError, + asyncio.TimeoutError, + ) as e: + # A refused redirect is already coded (network_unavailable) + # and fails only this connection. + from deepctl_core import transport + + mapped = transport.map_ws_open( + e, (self.api_key,), host=self.upstream_host, path=path ) - return - except (WebSocketException, OSError, asyncio.TimeoutError) as e: + if mapped is not None and mapped.code in _FATAL_CODES: + # The key itself was refused: every connection would fail + # the same way. Stop, and let the command report the coded + # error. Only the first refusal stops. + if self.fatal is None: + self.fatal = mapped + await self._stop_all() + return console.print( Text.from_markup( - f"[red]Upstream connection failed[/red] [{conn_id}]: " + f"[red]Deepgram connection failed[/red] [{conn_id}]: " ), - untrusted(str(e)), + untrusted(f"{mapped.error} ({mapped.code})" if mapped else str(e)), sep="", ) return diff --git a/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py b/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py index 34fa059d..4800b470 100644 --- a/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py +++ b/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py @@ -1,5 +1,6 @@ """Tests for debug probe command and proxy.""" +import asyncio import json from unittest.mock import Mock, patch @@ -236,8 +237,13 @@ async def target(reader, writer): stubs.append(srv) ports[name] = srv.sockets[0].getsockname()[1] - proxy = ProbeProxy(api_key="fa1ce0000000000000000000000000000000000d") - proxy._upstream_base = f"ws://127.0.0.1:{ports['origin']}" + proxy = ProbeProxy( + api_key="fa1ce0000000000000000000000000000000000d", + upstream_base=f"ws://127.0.0.1:{ports['origin']}", + upstream_headers={ + "Authorization": "Token fa1ce0000000000000000000000000000000000d" + }, + ) server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] try: @@ -256,7 +262,7 @@ async def target(reader, writer): out = io.StringIO() with patch( "deepctl_cmd_debug_probe.proxy.console", - Console(file=out, force_terminal=False, width=300), + Console(file=out, force_terminal=False, width=300, stderr=True), ): asyncio.run(run()) return origin_reqs, target_reqs, out.getvalue() @@ -273,7 +279,8 @@ def test_redirect_refused_and_nothing_reaches_the_target(self, location): assert target == [] assert len(origin) == 1 assert b"Token fa1ce0" in origin[0] - assert "Upstream connection refused" in text + assert "Deepgram connection failed" in text + assert "(network_unavailable)" in text assert "dg debug probe doesn't follow redirects, so no audio was sent." in ( " ".join(text.split()) ) @@ -320,6 +327,8 @@ async def run(self, port): auth = Mock() auth.get_api_key.return_value = "fa1ce0000000000000000000000000000000000d" + # The Transport's host check: the default host passes. + auth.check_base_url.return_value = "https://api.deepgram.com" with ( patch("deepctl_cmd_debug_probe.command.require_ffprobe", return_value=True), patch.object(proxymod.ProbeProxy, "run", run), @@ -353,7 +362,7 @@ def custom(signum, frame): # pragma: no cover - never called signal.signal(signal.SIGINT, old) assert result.status == "success" assert seen == [] - assert "Shutting down probe proxy" in capsys.readouterr().out + assert "Stopping: open connections can finish" in capsys.readouterr().err def test_ignored_sigint_stays_ignored(self): """A background job in a script starts with SIGINT ignored: kill -INT @@ -387,7 +396,7 @@ def hooks(proxy, loop, port): assert out["ignored"] is True if out["sigign"] is not None: # Linux: the SIGINT bit is still set assert int(out["sigign"], 16) & (1 << (2 - 1)) - assert "Shutting down probe proxy" not in proc.stdout + assert "Stopping: open connections can finish" not in proc.stderr def test_keyboard_interrupt_path_first_and_second_press(self, capsys): """Where the loop can't take SIGINT (Windows), Ctrl+C raises @@ -423,7 +432,7 @@ def abort(): assert time.monotonic() - t0 < 10 assert result.status == "success" assert calls == ["abort"] - assert "Shutting down probe proxy" in capsys.readouterr().out + assert "Stopping: open connections can finish" in capsys.readouterr().err async def _hold_connection(proxy, port): @@ -438,7 +447,7 @@ async def never(reader, writer): writer.close() hang = await asyncio.start_server(never, "127.0.0.1", 0) - proxy._upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" + proxy.upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" for _ in range(100): try: ws = await connect(f"ws://127.0.0.1:{port}/v1/listen") @@ -520,3 +529,213 @@ def test_localhost_host_allowed_any_case(self): ) def test_foreign_host_refused(self, host): assert self._status([("Host", host)]) == 403 + + +# ── The upstream comes from the Transport ── + +KEY = "fa1ce0000000000000000000000000000000000d" +_QUIET = __import__("logging").getLogger("test.probe.fake-upstream") +_QUIET.propagate = False +_QUIET.setLevel(__import__("logging").CRITICAL + 1) + + +async def _upstream(process_request, handler=None): + """A websockets server on a free loopback port. A refusal must come from + ``connection.respond()`` in ``process_request`` (websockets 13 runs the + handler for a hand-built Response).""" + from websockets.asyncio.server import serve + + async def echo(ws): + async for msg in ws: + await ws.send(msg) + + server = await serve( + handler or echo, + "127.0.0.1", + 0, + process_request=process_request, + logger=_QUIET, + ) + return server, next(iter(server.sockets)).getsockname()[1] + + +def _proxy_to(port): + proxy = ProbeProxy( + api_key=KEY, + upstream_base=f"http://127.0.0.1:{port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) + assert proxy.upstream_base == f"ws://127.0.0.1:{port}" + return proxy + + +async def _client_until_closed(port, target="/v1/listen?encoding=linear16"): + from websockets.asyncio.client import connect + from websockets.exceptions import ConnectionClosed, InvalidHandshake + + try: + async with connect(f"ws://127.0.0.1:{port}{target}", open_timeout=5) as ws: + await asyncio.wait_for(ws.recv(), 10) + except (ConnectionClosed, InvalidHandshake, OSError, asyncio.TimeoutError): + pass + + +def _run_proxy_against(process_request, *, connections=1): + """Serve the fake upstream and the proxy on loopback, connect + ``connections`` clients one after another, and run until the proxy stops + itself or the clients are done. Returns (proxy, seen upstream auth).""" + seen = [] + + def record(connection, request): + seen.append(request.headers.get("Authorization")) + return process_request(connection, request) + + async def main(): + upstream, up_port = await _upstream(record) + proxy = _proxy_to(up_port) + port = _free_port() + run = asyncio.ensure_future(proxy.run(port)) + await asyncio.sleep(0.1) + for _ in range(connections): + await _client_until_closed(port) + await asyncio.sleep(0.3) + proxy._stop.set() + await asyncio.wait_for(run, 20) + upstream.close() + await upstream.wait_closed() + return proxy + + return asyncio.run(main()), seen + + +def _refusing(status): + def process_request(connection, request): + return connection.respond(status, "{}") + + return process_request + + +@pytest.mark.parametrize(("status", "code"), [(401, "auth_invalid")]) +def test_refused_key_stops_the_proxy_with_its_code(status, code): + proxy, seen = _run_proxy_against(_refusing(status), connections=1) + assert proxy.fatal is not None + assert proxy.fatal.code == code + assert seen == [f"Token {KEY}"] + assert KEY not in proxy.fatal.error + + +@pytest.mark.parametrize("status", [400, 403, 404, 429, 500, 503]) +def test_other_refusals_fail_only_that_connection(status, capsys): + proxy, seen = _run_proxy_against(_refusing(status), connections=2) + assert proxy.fatal is None # the proxy kept serving + assert seen == [f"Token {KEY}"] * 2 + assert len(proxy.connections) == 2 + err = capsys.readouterr().err + assert err.count("Deepgram connection failed") == 2 + assert KEY not in err + if status == 404: + assert "/v1/listen" in err + + +def test_a_later_401_doesnt_stop_the_loop_again(): + """A second client refused after the first fatal 401 (here: while the + server shuts down) must not stop the proxy a second time.""" + proxy, seen = _run_proxy_against(_refusing(401), connections=2) + assert proxy.fatal is not None and proxy.fatal.code == "auth_invalid" + assert 1 <= len(seen) <= 2 + + +def test_fatal_stop_with_an_open_stream_is_fast(capsys): + """A 401 while another stream is open stops within seconds, with + no leftover tasks or loop exception-handler errors.""" + import time + + def process_request(connection, request): + if "mode=401" in request.path: + return connection.respond(401, "{}") + return None + + errors: list[dict] = [] + marks: dict[str, float] = {} + + async def main(): + asyncio.get_running_loop().set_exception_handler( + lambda _loop, ctx: errors.append(ctx) + ) + upstream, up_port = await _upstream(process_request) + proxy = _proxy_to(up_port) + port = _free_port() + run = asyncio.ensure_future(proxy.run(port)) + await asyncio.sleep(0.1) + + async def held(): + from websockets.asyncio.client import connect + from websockets.exceptions import ConnectionClosed + + async with connect(f"ws://127.0.0.1:{port}/v1/listen") as ws: + await ws.send("ping") + await asyncio.wait_for(ws.recv(), 5) + try: + await asyncio.wait_for(ws.recv(), 30) # until the proxy closes it + except ConnectionClosed: + pass + + holder = asyncio.ensure_future(held()) + await asyncio.sleep(0.5) + marks["refused"] = time.monotonic() + await _client_until_closed(port, "/v1/listen?mode=401") + await asyncio.wait_for(run, 30) + marks["stopped"] = time.monotonic() + await asyncio.wait_for(holder, 5) + upstream.close() + await upstream.wait_closed() + return proxy + + proxy = asyncio.run(main()) + assert proxy.fatal is not None and proxy.fatal.code == "auth_invalid" + assert marks["stopped"] - marks["refused"] < 10 + assert errors == [] + + +def test_open_timeout_fails_only_that_connection(monkeypatch, capsys): + from deepctl_core import transport + + monkeypatch.setattr(transport, "WS_OPEN_TIMEOUT_S", 0.3) + + async def hang(connection, request): + await asyncio.sleep(2) + + proxy, seen = _run_proxy_against(hang, connections=1) + assert proxy.fatal is None + assert "(timeout)" in capsys.readouterr().err + + +def test_default_upstream_is_the_hosts_default(): + from deepctl_core.hosts import DEFAULT_BASE_URL + + proxy = ProbeProxy(api_key=KEY) + assert proxy.upstream_base == DEFAULT_BASE_URL.replace("https://", "wss://") + assert proxy.upstream_headers == {} # no key without the command's check + + +def test_refused_host_gets_no_key(monkeypatch, tmp_path): + from deepctl_core import hosts, output + from deepctl_core.auth import AuthManager + from deepctl_core.config import Config + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv("DEEPGRAM_API_KEY", KEY) + config = Config(config_path=str(tmp_path / "none.yaml")) + built = [] + # handle() imports ProbeProxy from .proxy at call time, so the + # seam is the proxy module's name. + monkeypatch.setattr( + "deepctl_cmd_debug_probe.proxy.ProbeProxy", + lambda **kw: built.append(kw), + ) + with patch("deepctl_cmd_debug_probe.command.require_ffprobe", return_value=True): + with pytest.raises(hosts.CredentialHostError): + ProbeCommand().handle(config, AuthManager(config), Mock(), port=None) + assert built == [] diff --git a/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py b/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py index 43eeda4b..f23d9e98 100644 --- a/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py +++ b/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py @@ -1,7 +1,8 @@ """The probe proxy against real loopback servers: relay, refusals, run loop. The upstream is a websockets server on a free 127.0.0.1 port, reached -through the proxy's `_upstream_base` seam. Nothing is mocked. +through the proxy's `upstream_base`, with the Authorization the command's +Transport would pass. Nothing is mocked. """ from __future__ import annotations @@ -81,8 +82,12 @@ class Proxy: """A ProbeProxy listening on a free port, pointed at `upstream`.""" def __init__(self, upstream: FakeUpstream) -> None: - self.proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=10**9) - self.proxy._upstream_base = f"ws://127.0.0.1:{upstream.port}" + self.proxy = ProbeProxy( + api_key=KEY, + probe_interval_bytes=10**9, + upstream_base=f"ws://127.0.0.1:{upstream.port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) async def __aenter__(self) -> Proxy: self.server = await self.proxy.start(0) @@ -220,7 +225,7 @@ async def main() -> tuple[int, bytes, FakeUpstream]: status, body, up = asyncio.run(main()) assert (status, body) == (403, b"Forbidden.\n") assert up.handshakes == [] - out = " ".join(capsys.readouterr().out.split()) # rich wraps at 80 columns + out = " ".join(capsys.readouterr().err.split()) # rich wraps at 80 columns if "Origin" in headers: shown = headers["Origin"].replace("\t", "\\x09") assert ( @@ -269,15 +274,17 @@ async def main() -> None: await ws.close() asyncio.run(main()) - assert "path=/v1/listen[bold]x" in capsys.readouterr().out + assert "path=/v1/listen[bold]x" in capsys.readouterr().err -def test_upstream_401_reports_and_keeps_serving(capsys, caplog): +def test_upstream_403_reports_and_keeps_serving(capsys, caplog): + """A refusal other than 401 fails only that connection (only + auth_invalid stops the proxy; see test_probe's fatal tests).""" caplog.set_level(logging.DEBUG) async def main() -> list[tuple[int | None, str | None]]: closes = [] - async with FakeUpstream(status=401) as up, Proxy(up) as px: + async with FakeUpstream(status=403) as up, Proxy(up) as px: for _ in range(2): async with connect(f"ws://127.0.0.1:{px.port}/v1/listen") as ws: with pytest.raises(ConnectionClosed): @@ -288,8 +295,8 @@ async def main() -> list[tuple[int | None, str | None]]: assert asyncio.run(main()) == [(1000, ""), (1000, "")] out = capsys.readouterr() - assert out.out.count("Upstream connection failed") == 2 - assert "HTTP 401" in out.out + assert out.err.count("Deepgram connection failed") == 2 + assert "(auth_forbidden)" in out.err for text in (out.out, out.err, caplog.text): assert KEY not in text @@ -331,7 +338,7 @@ def test_run_stops_on_event_and_on_cancel_after_summaries(capsys): async def main() -> ProbeProxy: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" with __import__("socket").socket() as s: s.bind(("127.0.0.1", 0)) port = s.getsockname()[1] @@ -361,7 +368,7 @@ async def main() -> ProbeProxy: proxy = asyncio.run(main()) (stats,) = proxy.connections assert stats.duration_seconds is not None - out = capsys.readouterr().out + out = capsys.readouterr().err assert "Connection closed" in out assert "Sent: 1,000 bytes (1 binary, 0 text frames)" in out @@ -554,14 +561,14 @@ def test_snapshots_do_not_block_the_loop_and_stop_waits_for_summaries( proxymod, "get_ffprobe_path", lambda _config=None: _fake_ffprobe( - tmp_path, f"open({str(marker)!r}, 'w').close(); time.sleep(1.5)" + tmp_path, f"open({str(marker)!r}, 'w').close(); time.sleep(4)" ), ) async def main() -> tuple[float, ProbeProxy]: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=1) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -570,7 +577,7 @@ async def main() -> tuple[float, ProbeProxy]: await asyncio.sleep(0.2) a = await connect(f"ws://127.0.0.1:{port}/v1/listen") await a.recv() - await a.send(b"\x00" * 100) # starts a 1.5 s snapshot + await a.send(b"\x00" * 100) # starts a 4 s snapshot while not marker.exists(): await asyncio.sleep(0.01) t0 = time.monotonic() @@ -579,16 +586,16 @@ async def main() -> tuple[float, ProbeProxy]: await b.send("ping") assert await b.recv() == "ping" relayed_in = time.monotonic() - t0 - for _ in range(30): # a backlog: 30 more 1.5 s snapshots if drained + for _ in range(30): # a backlog: 30 more 4 s snapshots if drained await a.send(b"\x00" * 100) proxy._stop.set() # what the first Ctrl+C does, mid-snapshot - await asyncio.wait_for(task, 10) + await asyncio.wait_for(task, 20) return relayed_in, proxy relayed_in, proxy = asyncio.run(main()) - assert relayed_in < 1.0 + assert relayed_in < 2.5 assert all(c.duration_seconds is not None for c in proxy.connections) - out = capsys.readouterr().out + out = capsys.readouterr().err assert out.count("Connection closed") == 2 assert "Not forwarded (arrived while stopping): 3,000 bytes (30 frames)" in out @@ -601,7 +608,7 @@ def test_abort_ends_connections_without_waiting_for_close_timeout(capsys): async def main() -> float: async with _DeafUpstream() as up: proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -619,8 +626,8 @@ async def main() -> float: await asyncio.wait_for(task, 10) return time.monotonic() - t0 - assert asyncio.run(main()) < 3 - assert "Connection closed" in capsys.readouterr().out + assert asyncio.run(main()) < 10 + assert "Connection closed" in capsys.readouterr().err def test_abort_during_upstream_handshake(): @@ -631,7 +638,7 @@ async def never(reader: Any, writer: Any) -> None: hang = await asyncio.start_server(never, "127.0.0.1", 0) proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" + proxy.upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -675,7 +682,7 @@ def test_second_stop_kills_a_running_ffprobe_and_skips_the_final_probe( async def main() -> float: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=1) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -695,9 +702,9 @@ async def main() -> float: return time.monotonic() - t0 elapsed = asyncio.run(main()) - assert elapsed < 1.5 + assert elapsed < 4.0 assert len(list(runs.iterdir())) == 1 # no final probe - assert "Connection closed" in capsys.readouterr().out + assert "Connection closed" in capsys.readouterr().err def test_transcripts_print_literally_and_keep_unicode(capsys): @@ -714,7 +721,7 @@ def test_transcripts_print_literally_and_keep_unicode(capsys): ), stats, ) - out = capsys.readouterr().out + out = capsys.readouterr().err assert "[bold]x[/bold] :smile: \u3053\u3093\u306b\u3061\u306f[2J" in out assert "\x1b" not in out and "\x9b" not in out assert stats.transcripts == [transcript] @@ -732,6 +739,85 @@ def test_not_forwarded_line_counts(frames, size, line, capsys): stats = ProbeConnectionStats(connection_id="1234abcd") ProbeProxy(api_key=KEY)._print_connection_summary(stats, frames, size) - assert f" {line}\n" in capsys.readouterr().out + assert f" {line}\n" in capsys.readouterr().err ProbeProxy(api_key=KEY)._print_connection_summary(stats) - assert "Not forwarded" not in capsys.readouterr().out + assert "Not forwarded" not in capsys.readouterr().err + + +class _Redirector: + """A loopback upstream that answers every upgrade with a redirect and + counts the handshakes it gets and the Authorization values it saw.""" + + def __init__(self, status: int, location: str) -> None: + self.status = status + self.location = location + self.handshakes: list[str | None] = [] + + def _process(self, connection, request): # noqa: ANN001 + self.handshakes.append(request.headers.get("Authorization")) + response = connection.respond(self.status, "moved\n") + response.headers["Location"] = self.location.format(port=self.port) + return response + + async def __aenter__(self) -> "_Redirector": + from websockets.asyncio.server import serve + + self.server = await serve( + lambda ws: ws.close(), "127.0.0.1", 0, process_request=self._process + ) + self.port = self.server.sockets[0].getsockname()[1] + return self + + async def __aexit__(self, *exc: object) -> None: + self.server.close() + await self.server.wait_closed() + + +@pytest.mark.parametrize( + ("status", "cross_origin"), + [(302, True), (302, False), (301, True), (308, True), (307, False)], +) +def test_upstream_redirect_is_never_followed(status, cross_origin, capsys): + """A 3xx on the upstream upgrade fails only that connection with + network_unavailable; the redirect target sees no connection and the key + is sent once, to the configured upstream only.""" + + async def main(): + async with FakeUpstream() as target: + location = ( + f"ws://127.0.0.1:{target.port}/v1/listen" + if cross_origin + else "ws://127.0.0.1:{port}/v1/listen?again=1" + ) + async with _Redirector(status, location) as up: + proxy = ProbeProxy( + api_key=KEY, + upstream_base=f"ws://127.0.0.1:{up.port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) + server = await proxy.start(0) + port = server.sockets[0].getsockname()[1] + err = "" + try: + await _status(port, _upgrade(port)) + for _ in range(150): + err += capsys.readouterr().err + if "connection failed" in err: + break + await asyncio.sleep(0.02) + # A followed redirect would reach the target meanwhile. + await asyncio.sleep(0.2) + err += capsys.readouterr().err + finally: + server.close() + await server.wait_closed() + return err, up.handshakes, target.handshakes, proxy + + err, handshakes, target_handshakes, proxy = asyncio.run(main()) + assert handshakes == [f"Token {KEY}"] + assert target_handshakes == [] + assert proxy.fatal is None + flat = " ".join(err.split()) + assert "Deepgram connection failed" in flat + assert "(network_unavailable)" in flat + assert KEY not in err diff --git a/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py b/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py index 1e2a69c3..0e47f501 100644 --- a/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py +++ b/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py @@ -9,6 +9,9 @@ import click from deepctl_core import AuthenticationError, AuthManager, BaseGroupCommand, Config + +# Already loaded with BaseGroupCommand (deepctl_core.base_command imports it). +from deepctl_core.hosts import DEFAULT_BASE_URL from rich.console import Console from .fetcher import get_cached_manifest, get_or_fetch_script, refresh_manifest @@ -16,6 +19,9 @@ console = Console() +# The streaming endpoint the examples point at, built from the one default. +_LISTEN_WS_URL = DEFAULT_BASE_URL.replace("https://", "wss://", 1) + "/v1/listen" + class ToolkitCommand(BaseGroupCommand): """Runs field engineering scripts from deepgram/support-toolkit.""" @@ -35,8 +41,8 @@ class ToolkitCommand(BaseGroupCommand): examples = [ "dg debug toolkit refresh", - "dg debug toolkit latency wss://api.deepgram.com/v1/listen", - "dg debug toolkit latency wss://api.deepgram.com/v1/listen --duration=60", + f"dg debug toolkit latency {_LISTEN_WS_URL}", + f"dg debug toolkit latency {_LISTEN_WS_URL} --duration=60", ] def setup_commands(self) -> list[click.Command]: diff --git a/packages/deepctl-core/src/deepctl_core/messages.py b/packages/deepctl-core/src/deepctl_core/messages.py index 870b51c6..10639f8c 100644 --- a/packages/deepctl-core/src/deepctl_core/messages.py +++ b/packages/deepctl-core/src/deepctl_core/messages.py @@ -31,7 +31,7 @@ # The command comes last, with no trailing period, so it can be copied. E_FH_PROFILE_HINT = ( "If you trust {host}, add it to api.allowed_hosts in {user_config}. To " - "switch back to Deepgram, run: dg --base-url https://api.deepgram.com " + "switch back to Deepgram, run: dg --base-url https://api.deepgram.com " # host-literal: user text "login --profile {profile} --save-base-url --force-write" ) # Agent-mode hint shared by every foreign_host error. Never names a flag, From 0306f043657d8a87085e3e576cc1c4af30a200af Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 13:06:41 -0500 Subject: [PATCH 4/8] ci: keep the CLI smoke runs from sending telemetry release.yml installs the built wheels and the published release and runs deepctl in both. test.yml runs deepctl --version outside pytest, where conftest.py's opt-out doesn't reach. Turn telemetry off for all of them, and point test.yml's run at a dead loopback endpoint. --- .github/workflows/release.yml | 4 ++++ .github/workflows/test.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4400384..e3999d24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,10 @@ permissions: contents: write pull-requests: write +# Keep CI from sending telemetry from anything this workflow runs. +env: + DEEPCTL_TELEMETRY_DISABLED: "1" + jobs: release-please: name: Release Please diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed2ea345..c81b71d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,11 @@ jobs: run: | uv run pytest tests packages/deepctl-core/tests packages/deepctl-shared-utils/tests packages/deepctl-telemetry/tests packages/deepctl-cmd-api/tests packages/deepctl-cmd-login/tests packages/deepctl-cmd-projects/tests packages/deepctl-cmd-transcribe/tests packages/deepctl-cmd-usage/tests packages/deepctl-cmd-debug/tests packages/deepctl-cmd-debug-audio/tests packages/deepctl-cmd-debug-browser/tests packages/deepctl-cmd-debug-network/tests packages/deepctl-cmd-debug-toolkit/tests packages/deepctl-cmd-debug-probe/tests packages/deepctl-cmd-ffprobe/tests packages/deepctl-cmd-mcp/tests packages/deepctl-cmd-update/tests packages/deepctl-cmd-plugin/tests packages/deepctl-cmd-skills/tests packages/deepctl-cmd-init/tests packages/deepctl-cmd-models/tests packages/deepctl-cmd-speak/tests packages/deepctl-cmd-keys/tests packages/deepctl-cmd-read/tests packages/deepctl-cmd-listen/tests packages/deepctl-cmd-requests/tests packages/deepctl-cmd-billing/tests packages/deepctl-cmd-members/tests packages/deepctl-cmd-completion/tests packages/deepctl-plugin-example/tests --cov --cov-report=term-missing -v + # Outside pytest, so conftest.py's telemetry opt-out doesn't apply. - name: Test CLI installation + env: + DEEPCTL_TELEMETRY_DISABLED: "1" + DEEPCTL_TELEMETRY_DSN: "http://k@127.0.0.1:9/1" run: | uv run python -c "from deepctl.main import main; print('Import successful')" uv run deepctl --version From 9297ddc4ff55f40bc0cee8ee7ac447dc33b49903 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 13:07:59 -0500 Subject: [PATCH 5/8] ci: publish every release from release.yml with trusted publishing publish-on-release.yml rebuilt and published a tag when someone published a GitHub release by hand. Replace it with a tag input on release.yml's workflow_dispatch, so every PyPI upload comes from one workflow file. A dispatch with a tag skips release-please, checks that the tag is a vX.Y.Z published (not draft) GitHub release in the history of main, release/* or next/*, builds that tag, and refuses to go on unless the build holds the deepctl wheel for the tag's version. A release that release-please cuts builds the tag it just created, not the branch the run started on, and goes through the same version check. The build and install-test jobs now read the repository only and keep no credentials in the checkout. The publish job no longer reads a long-lived API token. It runs in the pypi environment with id-token: write, and each PyPI project accepts it as a trusted publisher bound to release.yml and that environment. Each upload sends every sub-package first and the root package last, in a second step that runs only if the first succeeded. The root pins every sub-package ==, so it is never on the index without them. A tag dispatch can also pick index: testpypi. That uploads the same build to TestPyPI from a separate testpypi environment, so the setup can be tried end to end before the first real OIDC release, and it advertises nothing. --- .github/workflows/publish-on-release.yml | 59 -------- .github/workflows/release.yml | 165 ++++++++++++++++++++++- 2 files changed, 162 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/publish-on-release.yml diff --git a/.github/workflows/publish-on-release.yml b/.github/workflows/publish-on-release.yml deleted file mode 100644 index 673795d3..00000000 --- a/.github/workflows/publish-on-release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Publish to PyPI on release - -# Fires when a human publishes a GitHub release with `gh release create -# v0.X.Y` (or via the GitHub UI) to recover from a stuck release-please -# state. Typical trigger: release-please aborts with `untagged, merged -# release PRs outstanding` because a previous merged release PR didn't -# get tagged, and `autorelease: pending` is stuck on it. -# -# Releases created by GITHUB_TOKEN (release-please's normal flow) do NOT -# trigger this workflow because GitHub Actions intentionally blocks that -# loop, so the standard release-please path keeps using release.yml's -# existing publish job. This workflow ONLY fires on human-driven -# recovery releases, so there is no double-publish race. -# -# Intentionally NO workflow_dispatch trigger. Manual deploys outside a -# real release-published event have shipped landing pages and packages -# that pointed at unreleased versions. Production publish must always -# be anchored to a real GitHub release artifact. - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - publish: - name: Build and publish to PyPI - runs-on: ubuntu-latest - # Only fire on root-package releases (v0.2.4, v1.0.0, …). - # Sub-package tags look like deepctl-cmd-listen-v0.0.3, which we skip; - # release-please's normal flow handles those via release.yml. - if: startsWith(github.event.release.tag_name, 'v') - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - with: - ref: ${{ github.event.release.tag_name }} - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.11" - - - name: Install build dependencies - run: pip install build twine - - - name: Build all packages - run: make build - - - name: Verify built packages - run: make verify-packages - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: dist/ - skip-existing: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3999d24..af67ace9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,27 @@ on: push: branches: - main + # A dispatch with no tag runs release-please, like a push. A dispatch + # with a tag rebuilds and publishes that existing release instead: the + # way to recover when a release was tagged but never reached PyPI. workflow_dispatch: + inputs: + tag: + description: >- + Existing root release tag (vX.Y.Z) to rebuild and publish. + Leave empty to run release-please. + type: string + required: false + default: "" + index: + description: >- + Where a tag dispatch publishes. testpypi is the dry run of the + trusted-publishing setup and advertises nothing. + type: choice + options: + - pypi + - testpypi + default: pypi permissions: contents: write @@ -30,6 +50,16 @@ jobs: token: ${{ github.token }} config-file: .github/release-please-config.json manifest-file: .github/.release-please-manifest.json + # Only a push or a plain dispatch cuts a release. A tag dispatch + # republishes an existing tag, and a TestPyPI dry run must not + # create a real release. + if: ${{ github.event_name == 'push' || (inputs.tag == '' && inputs.index != 'testpypi') }} + + - name: Refuse a TestPyPI dry run without a tag + if: ${{ github.event_name == 'workflow_dispatch' && inputs.index == 'testpypi' && inputs.tag == '' }} + run: | + echo "::error::a TestPyPI dry run needs a tag to publish" + exit 1 # The release PR's re-pin runs as two jobs so that no third-party code ever # runs next to a write credential: @@ -222,10 +252,54 @@ jobs: build: name: Build packages needs: release-please - if: ${{ needs.release-please.outputs.release_created }} + if: >- + ${{ needs.release-please.outputs.release_created == 'true' || + (github.event_name == 'workflow_dispatch' && inputs.tag != '') }} runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + persist-credentials: false + # A new release builds the tag release-please just created, not + # whatever the run's branch points at. A recovery checks out its + # tag below, and needs the branch history and its tags for that. + ref: ${{ (github.event_name != 'workflow_dispatch' || inputs.tag == '') && needs.release-please.outputs.tag_name || '' }} + fetch-depth: ${{ (github.event_name == 'workflow_dispatch' && inputs.tag != '') && '0' || '1' }} + + # Only a root release tag that is already a published GitHub release + # and is in the history of the branch this run was started on. The + # publish environments accept only main, release/* and next/*. + - name: Check out the recovery tag + if: ${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }} + env: + TAG: ${{ inputs.tag }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Checked before TAG is printed anywhere. + if ! [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::the tag input is not a root release tag (vX.Y.Z)"; exit 1 + fi + case "${GITHUB_REF}" in + refs/heads/main | refs/heads/release/* | refs/heads/next/*) ;; + *) echo "::error::start a recovery from main, release/* or next/*, not ${GITHUB_REF}"; exit 1 ;; + esac + if ! git rev-parse -q --verify "refs/tags/${TAG}^{commit}" >/dev/null; then + echo "::error::no tag ${TAG}"; exit 1 + fi + if ! git merge-base --is-ancestor "refs/tags/${TAG}^{commit}" "${GITHUB_SHA}"; then + echo "::error::${TAG} is not in the history of ${GITHUB_REF_NAME}"; exit 1 + fi + if ! DRAFT=$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft); then + echo "::error::${TAG} has no GitHub release"; exit 1 + fi + if [ "${DRAFT}" != "false" ]; then + echo "::error::${TAG}'s GitHub release is still a draft"; exit 1 + fi + git checkout --detach "refs/tags/${TAG}^{commit}" + echo "Rebuilding ${TAG} at $(git rev-parse HEAD)" - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -241,6 +315,20 @@ jobs: - name: Verify built packages run: make verify-packages + # A tag name can move; the version inside the build can't. Publish + # only if the tree that was built carries the tag's version. + - name: Check the build matches the release tag + if: ${{ startsWith(inputs.tag || needs.release-please.outputs.tag_name, 'v') }} + env: + TAG: ${{ inputs.tag || needs.release-please.outputs.tag_name }} + run: | + set -euo pipefail + if ! compgen -G "dist/deepctl-${TAG#v}-*.whl" >/dev/null; then + echo "::error::the build has no deepctl ${TAG#v} wheel; ${TAG} does not point at that release" + ls dist/ + exit 1 + fi + - name: Upload artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -251,8 +339,12 @@ jobs: name: Test installation needs: build runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -377,10 +469,20 @@ jobs: uses: ./.github/workflows/web-production.yml secrets: inherit + # Trusted Publishing: each of the 31 PyPI projects names this file and + # the pypi environment as its publisher, and PyPI swaps the job's OIDC + # token for a short-lived upload token. There is no API token to read. + # The pypi environment deploys only from main, release/* and next/*. publish: name: Publish to PyPI needs: [build, test] + if: ${{ github.event_name == 'push' || inputs.index == 'pypi' }} runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/deepctl/ + permissions: + id-token: write steps: - name: Download artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -388,13 +490,70 @@ jobs: name: dist path: dist/ - - name: Publish to PyPI + # The root package goes last: it pins every sub-package ==, so it must + # never be on the index before them. "deepctl-" sorts before + # "deepctl_", so one upload of dist/ would send it first. + - name: Hold the root package back + run: | + set -euo pipefail + mkdir root-dist + mv dist/deepctl-[0-9]* root-dist/ + ls root-dist/ + + - name: Publish the sub-packages to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/ + skip-existing: true + + - name: Publish the root package to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: root-dist/ + skip-existing: true + + # The same upload against TestPyPI, from its own environment so that its + # OIDC token can never match a production PyPI publisher. Nothing + # downstream needs this job, so a dry run advertises nothing. + publish-testpypi: + name: Publish to TestPyPI + needs: [build, test] + if: ${{ github.event_name == 'workflow_dispatch' && inputs.index == 'testpypi' }} + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/deepctl/ + permissions: + id-token: write + steps: + - name: Download artifacts + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + + # Root package last, as in publish. + - name: Hold the root package back + run: | + set -euo pipefail + mkdir root-dist + mv dist/deepctl-[0-9]* root-dist/ + ls root-dist/ + + - name: Publish the sub-packages to TestPyPI uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: - password: ${{ secrets.PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ packages-dir: dist/ skip-existing: true + - name: Publish the root package to TestPyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: root-dist/ + skip-existing: true + bump-brew-formula: name: Bump Homebrew formula needs: [release-please, publish, verify-published] From 785ff55098d07392292129c8f92cc420a297c1ed Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 13:09:58 -0500 Subject: [PATCH 6/8] fix(update): never wait for the update check The update, plugin-update and skills checks no longer join their threads at exit. A notice prints only if its check already finished; otherwise it is dropped and the daemon thread ends with the process. The checks now skip CI, agent and other non-interactive runs, dg mcp, a stderr that isn't a terminal, and DEEPCTL_NO_UPDATE_CHECK. httpx loads only inside the check thread. Each check records when it ran in the cache directory (update-check.json and plugin-update-check.json; 0600 files, replaced atomically). The record is written just before the request goes out, so a failed or unfinished request waits a full interval before the next attempt, and a run that ends earlier doesn't use the interval up. When no check is due and the saved answer names a newer version, the notice prints from it, at most once per interval (notified_at). The cache directory must be private: one this user owns with group or other access (a plain mkdir under umask 002) is tightened to 0700. If it is owned by someone else or can't be tightened, the checks don't run, with no network and no state. dg update keeps its last answer there too, and no longer writes the installation method or path. The old config keys are still read while no cache file exists, and are never written again. It shows the installation method by its plain name. --- .../src/deepctl_cmd_skills/startup_check.py | 70 ++- .../tests/unit/test_skills_command.py | 98 +++ .../src/deepctl_cmd_update/check_cache.py | 159 +++++ .../src/deepctl_cmd_update/command.py | 13 +- .../deepctl_cmd_update/plugin_update_check.py | 89 ++- .../src/deepctl_cmd_update/startup_check.py | 239 +++++--- .../src/deepctl_cmd_update/version_check.py | 67 +- .../tests/unit/test_plugin_update_check.py | 231 ++++--- .../tests/unit/test_startup_check.py | 572 ++++++++++++++---- .../tests/unit/test_update_command.py | 50 ++ .../tests/unit/test_version_check.py | 109 ++++ tests/unit/test_base_url_persistence.py | 27 +- 12 files changed, 1341 insertions(+), 383 deletions(-) create mode 100644 packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py diff --git a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py index 8d153221..35deb9c2 100644 --- a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py +++ b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py @@ -1,7 +1,11 @@ """First-run prompt to nudge users to set up AI assistant skills. -Checks for installed AI coding CLIs and prints a one-line notification -to stderr. Shows only once (uses a cache file to track). +Checks for installed AI coding CLIs in a daemon thread and prints a +one-line notification to stderr after the command, only if the thread has +already finished; dg never waits for it. Shows only once (a cache file +records that it was shown), and only to a person at a terminal: never in +CI, an AI agent session, under ``dg mcp``, with ``DEEPCTL_NO_UPDATE_CHECK`` +set, or when stderr isn't a terminal. """ from __future__ import annotations @@ -41,13 +45,44 @@ def _is_oneshot() -> bool: return bool(exe_path.startswith(pipx_cache)) +def _stderr_is_terminal() -> bool: + try: + return bool(sys.stderr is not None and sys.stderr.isatty()) + except (AttributeError, OSError, ValueError): + return False + + +def _is_agent() -> bool: + try: + from deepctl_core.output import is_agentic, is_ai_agent + except ImportError: + return False + return is_agentic() or is_ai_agent() + + +def _opted_out() -> bool: + """``DEEPCTL_NO_UPDATE_CHECK`` turns off every startup check.""" + value = os.environ.get("DEEPCTL_NO_UPDATE_CHECK", "").strip().lower() + return value not in ("", "0", "false", "no") + + +def _is_mcp() -> bool: + """This run is ``dg mcp`` (root options before it allowed).""" + try: + from deepctl_cmd_update.startup_check import _subcommand + except ImportError: + args = [a for a in sys.argv[1:] if not a.startswith("-")] + return bool(args) and args[0] == "mcp" + return _subcommand() == "mcp" + + def _already_prompted() -> bool: return _CACHE_FILE.exists() def _mark_prompted() -> None: try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) + _CACHE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) _CACHE_FILE.write_text("1") except OSError: pass @@ -66,7 +101,6 @@ def _background_detect(result: dict[str, bool]) -> None: detected = detect_ai_clis() if detected: result["should_prompt"] = True - _mark_prompted() except Exception: pass @@ -83,29 +117,42 @@ def check_and_notify(quiet: bool = False) -> None: after CLI execution to display the result. """ global _thread, _result + _thread = None _result = {} - if quiet or _is_ci() or _is_oneshot() or _already_prompted(): + if ( + quiet + or _opted_out() + or _is_ci() + or not _stderr_is_terminal() + or _is_agent() + or _is_oneshot() + or _is_mcp() + or _already_prompted() + ): return _thread = threading.Thread( target=_background_detect, args=(_result,), + name="deepctl-skills-check", daemon=True, ) _thread.start() def print_pending_notification() -> None: - """Print the skills prompt if AI CLIs were detected.""" + """Print the skills prompt if the check already found AI CLIs. + + Never waits: a check that is still running is dropped, and the prompt + is recorded as shown only once it is printed. + """ global _thread - if _thread is None: + thread, _thread = _thread, None + if thread is None or thread.is_alive(): return - _thread.join(timeout=2.0) - _thread = None - if _result.get("should_prompt"): try: sys.stderr.write( @@ -115,4 +162,5 @@ def print_pending_notification() -> None: except (BrokenPipeError, OSError, ValueError): # The output stream closed (e.g. an MCP host disconnected stdio # after `dg mcp` finished). The prompt is purely advisory. - pass + return + _mark_prompted() diff --git a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py index 700bc305..4567d8e3 100644 --- a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py +++ b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py @@ -965,6 +965,104 @@ def test_suppressed_when_quiet(self): startup_check.check_and_notify(quiet=True) assert startup_check._thread is None + def test_suppressed_when_stderr_is_not_a_terminal(self): + from deepctl_cmd_skills import startup_check + + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=False), + patch.object(startup_check, "_is_oneshot", return_value=False), + patch.object(startup_check, "_already_prompted", return_value=False), + patch.object(startup_check, "_stderr_is_terminal", return_value=False), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + @pytest.mark.parametrize("case", ["mcp", "opt-out"]) + def test_suppressed_for_mcp_and_opt_out(self, case, monkeypatch): + from deepctl_cmd_skills import startup_check + + monkeypatch.delenv("DEEPCTL_NO_UPDATE_CHECK", raising=False) + if case == "opt-out": + monkeypatch.setenv("DEEPCTL_NO_UPDATE_CHECK", "1") + mcp = False + else: + mcp = True + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=False), + patch.object(startup_check, "_is_oneshot", return_value=False), + patch.object(startup_check, "_already_prompted", return_value=False), + patch.object(startup_check, "_stderr_is_terminal", return_value=True), + patch.object(startup_check, "_is_mcp", return_value=mcp), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + def test_mcp_detected_through_root_options(self, monkeypatch): + from deepctl import _cli + from deepctl_cmd_skills import startup_check + + monkeypatch.setattr(_cli, "_run_args", ["--profile", "x", "mcp"]) + assert startup_check._is_mcp() is True + monkeypatch.setattr(_cli, "_run_args", ["models"]) + assert startup_check._is_mcp() is False + + def test_suppressed_for_an_agent(self): + from deepctl_cmd_skills import startup_check + + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=True), + patch.object(startup_check, "_stderr_is_terminal", return_value=True), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + def test_running_check_is_dropped_without_waiting(self): + import threading + + from deepctl_cmd_skills import startup_check + + release = threading.Event() + startup_check._result = {"should_prompt": True} + thread = threading.Thread(target=release.wait, daemon=True) + thread.start() + startup_check._thread = thread + try: + with ( + patch.object(thread, "join") as join, + patch.object(startup_check, "_mark_prompted") as mark, + patch.object(sys, "stderr", MagicMock()) as err, + ): + startup_check.print_pending_notification() + join.assert_not_called() + err.write.assert_not_called() + mark.assert_not_called() + finally: + release.set() + thread.join(timeout=5.0) + + def test_marked_only_once_shown(self): + import threading + + from deepctl_cmd_skills import startup_check + + startup_check._result = {"should_prompt": True} + startup_check._thread = threading.Thread(target=lambda: None) + startup_check._thread.start() + startup_check._thread.join() + with ( + patch.object(startup_check, "_mark_prompted") as mark, + patch.object(sys, "stderr", MagicMock()) as err, + ): + startup_check.print_pending_notification() + err.write.assert_called_once() + mark.assert_called_once() + @pytest.mark.parametrize( "error", [ diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py new file mode 100644 index 00000000..ecf7ef8a --- /dev/null +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py @@ -0,0 +1,159 @@ +"""When the update checks last ran, kept in dg's cache directory. + +Each check has one small JSON file there, ``update-check.json`` for dg itself +and ``plugin-update-check.json`` for plugins, holding ``checked_at`` (Unix +time), ``ok``, ``latest`` and ``source``, plus ``notified_at`` once a notice +was shown. A check writes its file just before it sends its request and again +when it gets an answer, so a check that fails or never finishes still waits a +full interval before the next attempt, while a run that ends before the +request goes out doesn't use up the interval. + +The user config file is never written here. The directory is 0700 and must be +a real directory owned by this user: :func:`usable` tightens one this user +owns that its group or others can reach (``mkdir`` under umask 002 leaves +0775), and reports anything else as unusable, in which case the checks don't +run at all. Files are 0600 and replaced atomically (``deepctl._cache``). A +file that isn't a regular 0600-style file owned by this user, or any error, +reads as "no state". +""" + +from __future__ import annotations + +import json +import os +import stat +import sys +import time +from typing import Any + +UPDATE_FILE = "update-check.json" +PLUGIN_FILE = "plugin-update-check.json" + + +def _root() -> str | None: + try: + from deepctl import _cache + except ImportError: + return None + return _cache.cache_root() + + +def usable() -> bool: + """Whether the checks may run: the cache directory exists (created 0700 + if missing), is a real directory owned by this user, and is private + (tightened to 0700 here if its group or others had access). + + ``False`` means: skip the checks this run, with no network and no state. + """ + root = _root() + if root is None: + return False + try: + os.makedirs(root, mode=0o700, exist_ok=True) + except OSError: + return False + if sys.platform == "win32": + return os.path.isdir(root) and not os.path.islink(root) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(root, flags) + except OSError: + return False + try: + info = os.fstat(fd) + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid(): + return False + if stat.S_IMODE(info.st_mode) & 0o077: + os.fchmod(fd, 0o700) + return not stat.S_IMODE(os.fstat(fd).st_mode) & 0o077 + return True + except OSError: + return False + finally: + os.close(fd) + + +def _directory() -> str | None: + """The cache directory, or ``None`` if it's missing, unsafe or unknown.""" + root = _root() + if root is None: + return None + from deepctl import _cache + + return root if _cache._safe_dir(root) else None + + +def read(name: str) -> dict[str, Any] | None: + """The state saved in ``name``, or ``None``.""" + directory = _directory() + if directory is None: + return None + from deepctl import _cache + + data = _cache._read_private(os.path.join(directory, name)) + if data is None: + return None + try: + state = json.loads(data) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def _save(name: str, state: dict[str, Any]) -> None: + directory = _directory() + if directory is None: + return + from deepctl import _cache + + try: + _cache._write_private(os.path.join(directory, name), json.dumps(state).encode()) + except OSError: + pass + + +def write(name: str, *, ok: bool, latest: str | None, source: str) -> None: + """Save the state of a check that ran now. Never raises.""" + _save( + name, + {"checked_at": time.time(), "ok": ok, "latest": latest, "source": source}, + ) + + +def mark_notified(name: str) -> None: + """Record that the notice for the saved state was shown now.""" + state = read(name) + if state is not None: + state["notified_at"] = time.time() + _save(name, state) + + +def _time(state: dict[str, Any] | None, key: str) -> float: + value = (state or {}).get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0 + return float(value) + + +def checked_at(state: dict[str, Any] | None) -> float: + """When the check in ``state`` ran (``0.0`` if unknown).""" + return _time(state, "checked_at") + + +def notified_at(state: dict[str, Any] | None) -> float: + """When the notice for ``state`` was last shown (``0.0`` if never).""" + return _time(state, "notified_at") + + +def elapsed(since: float, interval: float) -> bool: + """Whether at least ``interval`` seconds passed since ``since``. + + A time in the future (a clock that moved back) counts as passed. + """ + gone = time.time() - since + return not 0 <= gone < interval + + +def due(name: str, interval: float) -> bool: + """Whether at least ``interval`` seconds passed since the last check.""" + return elapsed(checked_at(read(name)), interval) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py index 10743c6d..675393ec 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py @@ -153,15 +153,8 @@ def handle( detector = InstallationDetector() install_info = detector.detect() - # Store installation info for future use — serialize enum to its string value - config._set_config_value( - "update.installation_method", install_info.method.value - ) - config._set_config_value("update.installation_path", install_info.path) - config.save(implicit=True) - # Display installation info - console.print(f"Installation method: [cyan]{install_info.method}[/cyan]") + console.print(f"Installation method: [cyan]{install_info.method.value}[/cyan]") console.print(f"Installation path: [dim]{install_info.path}[/dim]") if install_info.virtual_env: console.print("[yellow]Virtual environment detected[/yellow]") @@ -219,10 +212,6 @@ def handle( f"Successfully updated to version {version_info.latest_version}" ) - # Clear version cache - config._set_config_value("update.cached_version_info", None) - config.save(implicit=True) - return UpdateResult( success=True, message=f"Successfully updated to version {version_info.latest_version}", diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py index e490cb3b..d4783964 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py @@ -2,7 +2,10 @@ Discovers community plugins (excluding first-party packages) and checks PyPI for newer versions. Runs in a daemon thread alongside the core -update check and prints notifications to stderr after CLI execution. +update check, under the same rules: skipped when the core check is, +at most once per interval with its state in +``/plugin-update-check.json``, and its notice prints after CLI +execution only if the thread already finished (dg never waits for it). """ from __future__ import annotations @@ -10,19 +13,17 @@ import importlib.metadata import sys import threading -import time -from pathlib import Path from typing import Any -from .startup_check import _get_check_interval_seconds, _is_ci, _is_oneshot +from . import check_cache +from .startup_check import _get_check_interval_seconds, suppressed + +THREAD_NAME = "deepctl-plugin-update-check" # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- -_CACHE_DIR = Path.home() / ".cache" / "deepctl" -_PLUGIN_CACHE_FILE = _CACHE_DIR / "last_plugin_version_check" - # First-party packages are version-locked to deepctl — never check these. _EXCLUDED_PREFIXES = ("deepctl-cmd-", "deepctl-core", "deepctl-shared-") _EXCLUDED_EXACT = frozenset({"deepctl", "deepctl-plugin-example"}) @@ -42,27 +43,6 @@ def _is_excluded(name: str) -> bool: return any(name.startswith(prefix) for prefix in _EXCLUDED_PREFIXES) -def _read_plugin_cache_timestamp() -> float: - """Read the last plugin-check timestamp from the cache file. - - Returns: - Unix timestamp of the last check, or ``0.0`` if not available. - """ - try: - return float(_PLUGIN_CACHE_FILE.read_text().strip()) - except (FileNotFoundError, ValueError, OSError): - return 0.0 - - -def _write_plugin_cache_timestamp() -> None: - """Persist the current time to the plugin cache file.""" - try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) - _PLUGIN_CACHE_FILE.write_text(str(time.time())) - except OSError: - pass - - # --------------------------------------------------------------------------- # Discovery # --------------------------------------------------------------------------- @@ -131,9 +111,13 @@ def _discover_community_plugins() -> dict[str, str]: def _check_pypi_versions( plugins: dict[str, str], + record: bool = False, ) -> list[dict[str, str]]: """Query PyPI for each plugin and return those with available updates. + With ``record``, the attempt is saved once the client is built, just + before the first request goes out. + Returns: List of ``{"name": ..., "current": ..., "latest": ...}`` dicts for plugins where a newer version exists on PyPI. @@ -148,6 +132,10 @@ def _check_pypi_versions( from packaging import version as pkg_version with httpx.Client(timeout=5.0) as client: + if record: + check_cache.write( + check_cache.PLUGIN_FILE, ok=False, latest=None, source="pypi" + ) for name, current_ver in list(plugins.items())[:_MAX_PLUGINS_TO_CHECK]: try: resp = client.get(f"https://pypi.org/pypi/{name}/json") @@ -177,19 +165,21 @@ def _check_pypi_versions( def _background_plugin_check(result: dict[str, Any]) -> None: - """Run plugin discovery + PyPI check (called inside a daemon thread).""" + """Run plugin discovery + PyPI check (called inside a daemon thread). + + The attempt is saved just before the first request goes out (see + :func:`_check_pypi_versions`), so one that fails or never finishes still + counts as this interval's check, while a run that ends before then + doesn't. + """ try: plugins = _discover_community_plugins() - if not plugins: - return - - updates = _check_pypi_versions(plugins) - if updates: - result["updates"] = updates - - _write_plugin_cache_timestamp() + updates = _check_pypi_versions(plugins, record=True) if plugins else [] except Exception: - pass + return + check_cache.write(check_cache.PLUGIN_FILE, ok=True, latest=None, source="pypi") + if updates: + result["updates"] = updates # --------------------------------------------------------------------------- @@ -216,22 +206,27 @@ def check_plugins_and_notify(quiet: bool = False) -> None: quiet: If True, suppress the check entirely. """ global _thread, _result + _thread = None _result = {} - if quiet or _is_ci() or _is_oneshot(): + if suppressed(quiet): return interval = _get_check_interval_seconds(quiet) if interval is None: return - last_check = _read_plugin_cache_timestamp() - if time.time() - last_check < interval: + # A cache directory that can't be made private: no check, no state. + if not check_cache.usable(): + return + + if not check_cache.due(check_cache.PLUGIN_FILE, interval): return _thread = threading.Thread( target=_background_plugin_check, args=(_result,), + name=THREAD_NAME, daemon=True, ) _thread.start() @@ -240,18 +235,16 @@ def check_plugins_and_notify(quiet: bool = False) -> None: def print_pending_plugin_notifications() -> None: """Print plugin update notifications if any are pending. - Call this **after** CLI execution completes. It joins the background - thread (with a short timeout) and, if updates were found, prints - ANSI-colored lines to stderr. + Call this **after** CLI execution completes. It never waits: if the + thread already finished and found updates, it prints ANSI-colored + lines to stderr; a check that is still running is dropped. """ global _thread - if _thread is None: + thread, _thread = _thread, None + if thread is None or thread.is_alive(): return - _thread.join(timeout=3.0) - _thread = None - updates = _result.get("updates") if not updates: return diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py index 889bab17..9d6693db 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py @@ -1,8 +1,23 @@ """Background startup update check for deepctl. -Runs a non-intrusive version check in a daemon thread during normal CLI -execution, then prints a one-line notification to stderr *after* the -command output completes. +Runs a version check in a daemon thread while the command runs. When the +command is done, a one-line notice goes to stderr only if the thread has +already finished; dg never waits for it. A check that hasn't finished is +dropped, and the interpreter stops the thread at exit. + +The check runs at most once per interval (daily by default). Its state lives +in ``/update-check.json`` (``check_cache``), written just before the +request goes out and again with the answer, so a failing or slow index is +tried again only after a full interval. When no check is due and the saved +answer names a newer version, the notice prints from it, at most once per +interval (``notified_at``), so an answer that came in too late for its own +run still reaches the person. The user config file is never written. When +the cache directory can't be made private, the check doesn't run at all. + +The check is skipped entirely under ``--quiet``, in CI, in an AI agent or +other non-interactive session, under ``dg mcp``, when stderr isn't a +terminal, for one-shot runs (uvx, pipx run, npx) and when +``DEEPCTL_NO_UPDATE_CHECK`` is set. """ from __future__ import annotations @@ -11,12 +26,13 @@ import os import sys import threading -import time -from pathlib import Path from typing import Any -# Sentinel used when the background check is suppressed or has no result -_NO_RESULT: dict[str, Any] = {} +from . import check_cache + +PYPI_URL = "https://pypi.org/pypi/deepctl/json" +NO_UPDATE_CHECK_ENV_VAR = "DEEPCTL_NO_UPDATE_CHECK" +THREAD_NAME = "deepctl-update-check" # Environment variables that indicate a CI environment _CI_ENV_VARS = ( @@ -31,11 +47,6 @@ "CODEBUILD_BUILD_ID", ) -# Cache file for last-check timestamp (separate from YAML config to -# avoid race conditions) -_CACHE_DIR = Path.home() / ".cache" / "deepctl" -_CACHE_FILE = _CACHE_DIR / "last_version_check" - def _is_ci() -> bool: """Return True if running inside a CI environment.""" @@ -48,31 +59,67 @@ def _is_oneshot() -> bool: if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: return True # pipx run + from pathlib import Path + exe_path = str(Path(sys.executable).resolve()) pipx_home = os.environ.get("PIPX_HOME", str(Path.home() / ".local" / "pipx")) pipx_cache = str(Path(pipx_home) / ".cache") return bool(exe_path.startswith(pipx_cache)) -def _read_cache_timestamp() -> float: - """Read the last-check timestamp from the cache file. +def _opted_out() -> bool: + """``DEEPCTL_NO_UPDATE_CHECK`` is set to anything but empty, 0 or false.""" + value = os.environ.get(NO_UPDATE_CHECK_ENV_VAR, "").strip().lower() + return value not in ("", "0", "false", "no") - Returns: - Unix timestamp of the last check, or ``0.0`` if not available. - """ + +def _stderr_is_terminal() -> bool: try: - return float(_CACHE_FILE.read_text().strip()) - except (FileNotFoundError, ValueError, OSError): - return 0.0 + return bool(sys.stderr is not None and sys.stderr.isatty()) + except (AttributeError, OSError, ValueError): + return False -def _write_cache_timestamp() -> None: - """Persist the current time to the cache file.""" +def _is_agent() -> bool: + """An AI agent or another non-interactive session (deepctl_core's rules).""" try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) - _CACHE_FILE.write_text(str(time.time())) - except OSError: - pass + from deepctl_core.output import is_agentic, is_ai_agent + except ImportError: + return False + return is_agentic() or is_ai_agent() + + +def _subcommand() -> str | None: + """The command this run executes (``mcp`` for ``dg -q mcp``), if known.""" + cli = sys.modules.get("deepctl._cli") + args = getattr(cli, "_run_args", None) + if args is None: + args = sys.argv[1:] + try: + flag_opts = cli._root_flag_opts() if cli is not None else set() + i = 0 + while i < len(args) and args[i].startswith("-"): + takes_value = cli is not None and cli._takes_next_token(args[i], flag_opts) + i += 2 if takes_value else 1 + except Exception: + return None + return args[i] if i < len(args) else None + + +def suppressed(quiet: bool) -> bool: + """Whether this run must not check for updates at all. + + Shared by the plugin-update check. Cheap: no config or network access. + """ + return ( + quiet + or _opted_out() + or _is_ci() + or not _stderr_is_terminal() + or _is_agent() + or _is_oneshot() + or _subcommand() == "mcp" + ) def _get_check_interval_seconds(quiet: bool) -> float | None: @@ -101,43 +148,78 @@ def _get_check_interval_seconds(quiet: bool) -> float | None: def _background_check(current_version: str, result: dict[str, Any]) -> None: - """Run the version check synchronously (called inside a thread).""" + """Run the version check synchronously (called inside a thread). + + The attempt is saved just before the request goes out, so one that + fails or never finishes still counts as this interval's check, while a + run that ends before the request (imports take a while) doesn't. + """ try: + # httpx loads here, in the thread, never on the main path. Building + # the client (CA bundle, proxy settings) also comes before the record. import httpx - - response = httpx.get( - "https://pypi.org/pypi/deepctl/json", - timeout=5.0, - ) - response.raise_for_status() - data = response.json() - latest_version: str = data["info"]["version"] - from packaging import version as pkg_version - current = pkg_version.parse(current_version) - latest = pkg_version.parse(latest_version) + client = httpx.Client(timeout=5.0) + except Exception: + return + with client: + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + try: + response = client.get(PYPI_URL) + response.raise_for_status() + latest_version = response.json()["info"]["version"] + if not isinstance(latest_version, str): + return + newer = pkg_version.parse(latest_version) > pkg_version.parse( + current_version + ) + except Exception: + # Best effort: the failure is already recorded above. + return - if latest > current: - result["latest"] = latest_version - result["current"] = current_version + check_cache.write( + check_cache.UPDATE_FILE, ok=True, latest=latest_version, source="pypi" + ) + if newer: + result["latest"] = latest_version + result["current"] = current_version + + +def _newer_saved( + state: dict[str, Any] | None, current_version: str, interval: float +) -> tuple[str, str] | None: + """``(current, latest)`` when the saved answer names a newer version + whose notice wasn't shown within ``interval``.""" + if not state or state.get("ok") is not True: + return None + latest = state.get("latest") + if not isinstance(latest, str): + return None + if not check_cache.elapsed(check_cache.notified_at(state), interval): + return None + try: + from packaging import version as pkg_version - _write_cache_timestamp() + if pkg_version.parse(latest) <= pkg_version.parse(current_version): + return None except Exception: - # Silently swallow — this is best-effort - pass + return None + return current_version, latest # Module-level state for the background thread _thread: threading.Thread | None = None _result: dict[str, Any] = {} +# A notice from the saved answer, when no check ran this time. +_saved_notice: tuple[str, str] | None = None def check_and_notify( current_version: str | None = None, quiet: bool = False, ) -> None: - """Start a background version check. + """Start a background version check, unless this run skips it. Call this **before** CLI execution. It spawns a daemon thread that runs concurrently. Call :func:`print_pending_notification` after CLI @@ -148,20 +230,20 @@ def check_and_notify( detected automatically from package metadata. quiet: If True, suppress the check entirely. """ - global _thread, _result + global _thread, _result, _saved_notice + _thread = None _result = {} + _saved_notice = None - # Suppress in quiet mode, CI, or one-shot execution - if quiet or _is_ci() or _is_oneshot(): + if suppressed(quiet): return - # Check interval interval = _get_check_interval_seconds(quiet) if interval is None: return - last_check = _read_cache_timestamp() - if time.time() - last_check < interval: + # A cache directory that can't be made private: no check, no state. + if not check_cache.usable(): return # Resolve current version @@ -171,39 +253,50 @@ def check_and_notify( except importlib.metadata.PackageNotFoundError: return + state = check_cache.read(check_cache.UPDATE_FILE) + if not check_cache.elapsed(check_cache.checked_at(state), interval): + _saved_notice = _newer_saved(state, current_version, interval) + return + _thread = threading.Thread( target=_background_check, args=(current_version, _result), + name=THREAD_NAME, daemon=True, ) _thread.start() def print_pending_notification() -> None: - """Print the update notification if one is pending. + """Print the update notice if the check already found one. - Call this **after** CLI execution completes. It joins the background - thread (with a short timeout) and, if an update was found, prints a - single ANSI-colored line to stderr. + Call this **after** CLI execution completes. It never waits: a check + that is still running is dropped (its answer, once saved, gives the + notice on a later run). Without a check this run, a saved answer that + names a newer version prints instead, at most once per interval. """ - global _thread + global _thread, _saved_notice - if _thread is None: + thread, _thread = _thread, None + notice, _saved_notice = _saved_notice, None + if thread is not None: + if thread.is_alive(): + return + latest = _result.get("latest") + current = _result.get("current") + notice = (current, latest) if latest and current else None + if notice is None: return - _thread.join(timeout=2.0) - _thread = None - - latest = _result.get("latest") - current = _result.get("current") - if latest and current: - try: - sys.stderr.write( - f"\n\033[33m Update available: {current} → {latest}" - f" — run \033[1mdg update\033[0m\033[33m to upgrade\033[0m\n\n" - ) - except (BrokenPipeError, OSError, ValueError): - # The output stream closed (e.g. an MCP host disconnected stdio - # after `dg mcp` finished). Nothing useful to do — the notification - # is purely advisory. - pass + current, latest = notice + try: + sys.stderr.write( + f"\n\033[33m Update available: {current} → {latest}" + f" — run \033[1mdg update\033[0m\033[33m to upgrade\033[0m\n\n" + ) + except (BrokenPipeError, OSError, ValueError): + # The output stream closed (e.g. an MCP host disconnected stdio + # after `dg mcp` finished). Nothing useful to do — the notification + # is purely advisory. + return + check_cache.mark_notified(check_cache.UPDATE_FILE) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py index ce3e5df5..f67e39ed 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py @@ -1,11 +1,21 @@ -"""Version checking functionality for deepctl.""" +"""Version checking functionality for deepctl. + +The last check's result lives in ``/update-check.json`` (see +``check_cache``), shared with the startup check. Versions before 0.5.0 kept +it in the user config (``update.last_check`` and +``update.cached_version_info``); those keys are still read while no cache +file exists, and never written. +""" from datetime import datetime, timedelta, timezone +from typing import Any from deepctl_core import Config from packaging import version from pydantic import BaseModel, Field +from . import check_cache + class VersionInfo(BaseModel): """Version information from PyPI.""" @@ -44,6 +54,8 @@ def __init__(self, config: Config, current_version: str = "0.0.0"): """ self.config = config self.current_version = current_version + # Where the last successful answer came from ("pypi" or "github"). + self._source = "pypi" def _get_cache_duration(self) -> timedelta | None: """Get the cache duration based on ``check_frequency`` config. @@ -80,6 +92,7 @@ async def check_version( return cached_info # Choose version source + self._source = "pypi" if install_method == "homebrew": version_info = await self._check_github_releases() else: @@ -89,6 +102,10 @@ async def check_version( self._cache_info(version_info) return version_info + check_cache.write( + check_cache.UPDATE_FILE, ok=False, latest=None, source=self._source + ) + # On error, return current version with no update return VersionInfo( current_version=self.current_version, @@ -174,6 +191,7 @@ async def _check_github_releases(self) -> VersionInfo | None: ) release_notes_url = data.get("html_url") + self._source = "github" current = version.parse(self.current_version) latest = version.parse(latest_version) @@ -188,6 +206,7 @@ async def _check_github_releases(self) -> VersionInfo | None: ) except Exception: # Fall back to PyPI on GitHub API failure + self._source = "pypi" return await self._check_pypi() def should_check(self) -> bool: @@ -206,7 +225,14 @@ def should_check(self) -> bool: # "never" — always skip return False - # Check last check timestamp + state = self._saved_state() + if state is not None: + elapsed = datetime.now(timezone.utc).timestamp() - check_cache.checked_at( + state + ) + return not 0 <= elapsed < cache_duration.total_seconds() + + # No cache file yet: the timestamp older versions kept in the config last_check = self.config.get("update.last_check") if last_check: try: @@ -222,12 +248,32 @@ def should_check(self) -> bool: return True + @staticmethod + def _saved_state() -> dict[str, Any] | None: + return check_cache.read(check_cache.UPDATE_FILE) + def _get_cached_info(self) -> VersionInfo | None: - """Get cached version info from config. + """Get the cached version info. Returns: Cached version info or None """ + state = self._saved_state() + if state is not None: + latest = state.get("latest") + if not state.get("ok") or not isinstance(latest, str): + return None + try: + available = version.parse(latest) > version.parse(self.current_version) + except version.InvalidVersion: + return None + return VersionInfo( + current_version=self.current_version, + latest_version=latest, + update_available=available, + ) + + # No cache file yet: what older versions kept in the config cached_data = self.config.get("update.cached_version_info") if cached_data: try: @@ -237,18 +283,19 @@ def _get_cached_info(self) -> VersionInfo | None: return None def _cache_info(self, info: VersionInfo) -> None: - """Cache version info in config. + """Save the answer to the cache file (never to the user config). Args: info: Version info to cache """ - self.config._set_config_value( - "update.last_check", datetime.now(timezone.utc).isoformat() - ) - self.config._set_config_value( - "update.cached_version_info", info.model_dump(mode="json") + if not check_cache.usable(): + return + check_cache.write( + check_cache.UPDATE_FILE, + ok=True, + latest=info.latest_version, + source=self._source, ) - self.config.save(implicit=True) def format_version_message(info: VersionInfo) -> str: diff --git a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py index 9de3b700..0c5cbc08 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py @@ -3,8 +3,6 @@ import io import sys import threading -import time -from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -13,12 +11,36 @@ _check_pypi_versions, _discover_community_plugins, _is_excluded, - _read_plugin_cache_timestamp, - _write_plugin_cache_timestamp, check_plugins_and_notify, print_pending_plugin_notifications, ) +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture +def check_allowed(monkeypatch, cache_dir): + """A run where the startup checks may start: an interactive person at a + terminal, outside CI and ``dg mcp``. Returns the cache directory.""" + import deepctl_cmd_update.startup_check as core + + monkeypatch.delenv(core.NO_UPDATE_CHECK_ENV_VAR, raising=False) + monkeypatch.setattr(core, "_is_ci", lambda: False) + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: True) + monkeypatch.setattr(core, "_is_agent", lambda: False) + monkeypatch.setattr(core, "_is_oneshot", lambda: False) + monkeypatch.setattr(core, "_subcommand", lambda: None) + monkeypatch.setattr(core, "_get_check_interval_seconds", lambda quiet: 86400.0) + return cache_dir + + # --------------------------------------------------------------------------- # _is_excluded # --------------------------------------------------------------------------- @@ -49,39 +71,6 @@ def test_arbitrary_community_plugin_not_excluded(self): assert _is_excluded("my-deepctl-addon") is False -# --------------------------------------------------------------------------- -# Cache timestamp -# --------------------------------------------------------------------------- - - -class TestPluginCacheTimestamp: - """Test file-based plugin cache timestamp.""" - - def test_read_missing_returns_zero(self, tmp_path): - with patch( - "deepctl_cmd_update.plugin_update_check._PLUGIN_CACHE_FILE", - tmp_path / "nonexistent", - ): - assert _read_plugin_cache_timestamp() == 0.0 - - def test_write_then_read_roundtrip(self, tmp_path): - cache_file = tmp_path / "last_plugin_version_check" - with ( - patch( - "deepctl_cmd_update.plugin_update_check._PLUGIN_CACHE_FILE", - cache_file, - ), - patch( - "deepctl_cmd_update.plugin_update_check._CACHE_DIR", - tmp_path, - ), - ): - _write_plugin_cache_timestamp() - ts = _read_plugin_cache_timestamp() - assert ts > 0.0 - assert abs(ts - time.time()) < 5.0 - - # --------------------------------------------------------------------------- # Discovery # --------------------------------------------------------------------------- @@ -254,43 +243,45 @@ def test_empty_plugins_returns_empty(self): class TestCheckPluginsAndNotify: """Test the plugin check lifecycle.""" - @patch.dict("os.environ", {"CI": "true"}, clear=False) - def test_suppressed_in_ci(self): + @pytest.fixture(autouse=True) + def _reset(self): import deepctl_cmd_update.plugin_update_check as mod mod._thread = None mod._result = {} + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + + def test_suppressed_with_the_core_check(self, check_allowed, monkeypatch): + import deepctl_cmd_update.plugin_update_check as mod + import deepctl_cmd_update.startup_check as core + + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: False) + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() + assert not check_allowed.exists() + + def test_suppressed_in_ci(self, check_allowed, monkeypatch): + import deepctl_cmd_update.plugin_update_check as mod + import deepctl_cmd_update.startup_check as core + + monkeypatch.setattr(core, "_is_ci", lambda: True) check_plugins_and_notify() assert mod._thread is None - def test_suppressed_in_quiet_mode(self): + def test_suppressed_in_quiet_mode(self, check_allowed): import deepctl_cmd_update.plugin_update_check as mod - mod._thread = None - mod._result = {} check_plugins_and_notify(quiet=True) assert mod._thread is None - @patch("deepctl_cmd_update.plugin_update_check._is_ci", return_value=False) - @patch( - "deepctl_cmd_update.plugin_update_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._get_check_interval_seconds", - return_value=86400.0, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._read_plugin_cache_timestamp", - return_value=0.0, - ) - def test_fresh_cache_spawns_thread(self, *_mocks): - """Cache expired → thread should be spawned.""" + def test_due_spawns_thread(self, check_allowed): + """No check within the interval → thread should be spawned.""" import deepctl_cmd_update.plugin_update_check as mod - mod._thread = None - mod._result = {} - with patch( "deepctl_cmd_update.plugin_update_check._background_plugin_check", side_effect=lambda res: None, @@ -298,30 +289,45 @@ def test_fresh_cache_spawns_thread(self, *_mocks): check_plugins_and_notify() assert mod._thread is not None assert mod._thread.daemon is True - mod._thread.join(timeout=2.0) + assert mod._thread.name == mod.THREAD_NAME - @patch("deepctl_cmd_update.plugin_update_check._is_ci", return_value=False) - @patch( - "deepctl_cmd_update.plugin_update_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._get_check_interval_seconds", - return_value=86400.0, - ) - def test_fresh_cache_suppresses_thread(self, *_mocks): - """Cache still fresh → no thread should be spawned.""" + def test_failure_waits_a_day(self, check_allowed): + """A failed check is recorded; the next run makes no attempt.""" import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache - mod._thread = None - mod._result = {} + seen = {} - with patch( - "deepctl_cmd_update.plugin_update_check._read_plugin_cache_timestamp", - return_value=time.time(), # just checked + def unreachable(*args, **kwargs): + seen["state"] = check_cache.read(check_cache.PLUGIN_FILE) + raise OSError("unreachable") + + with ( + patch.object( + mod, "_discover_community_plugins", return_value={"deepctl-x": "1"} + ), + patch("httpx.Client.get", side_effect=unreachable), ): check_plugins_and_notify() - assert mod._thread is None + mod._thread.join(timeout=5.0) + # Recorded before the request went out. + assert seen["state"] is not None and seen["state"]["ok"] is False + assert check_cache.read(check_cache.PLUGIN_FILE) is not None + + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() + + def test_success_is_recorded(self, check_allowed): + import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache + + with patch.object(mod, "_discover_community_plugins", return_value={}): + check_plugins_and_notify() + mod._thread.join(timeout=5.0) + state = check_cache.read(check_cache.PLUGIN_FILE) + assert state is not None and state["ok"] is True + assert check_cache.read(check_cache.UPDATE_FILE) is None # --------------------------------------------------------------------------- @@ -436,20 +442,63 @@ def test_no_updates_no_output(self, capsys): captured = capsys.readouterr() assert captured.err == "" - def test_thread_timeout_handling(self): - """If the thread is still running after timeout, no crash.""" + def test_running_check_is_dropped_without_waiting(self): + """A check still running at exit is never joined.""" import deepctl_cmd_update.plugin_update_check as mod - # Create a thread that would take longer than the timeout - mod._result = {} - mod._thread = threading.Thread(target=lambda: time.sleep(10), daemon=True) - mod._thread.start() + release = threading.Event() + mod._result = { + "updates": [{"name": "deepctl-plugin-x", "current": "1", "latest": "2"}] + } + thread = threading.Thread(target=release.wait, daemon=True) + thread.start() + mod._thread = thread + try: + with patch.object(thread, "join") as join: + stderr_capture = io.StringIO() + with patch.object(sys, "stderr", stderr_capture): + print_pending_plugin_notifications() + join.assert_not_called() + assert stderr_capture.getvalue() == "" + assert mod._thread is None + finally: + release.set() + thread.join(timeout=5.0) + - # Should not raise — just returns with no output - with patch.object(mod._thread, "join", side_effect=lambda timeout=None: None): - stderr_capture = io.StringIO() - with patch.object(sys, "stderr", stderr_capture): - print_pending_plugin_notifications() - output = stderr_capture.getvalue() - assert output == "" +class TestPluginAttemptRecording: + @pytest.fixture(autouse=True) + def _reset(self): + import deepctl_cmd_update.plugin_update_check as mod + + mod._thread = None + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + + def test_nothing_recorded_before_the_request(self, check_allowed, monkeypatch): + """A failure before any request (here: discovery) uses up nothing.""" + import deepctl_cmd_update.plugin_update_check as mod + + with patch.object( + mod, "_discover_community_plugins", side_effect=RuntimeError("boom") + ): + check_plugins_and_notify() + mod._thread.join(timeout=5.0) + assert not (check_allowed / "plugin-update-check.json").exists() + + def test_unusable_cache_dir_skips(self, check_allowed, monkeypatch): + import os + + import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache + + check_allowed.mkdir(parents=True, mode=0o755) + monkeypatch.setattr( + check_cache.os, "getuid", lambda: os.stat(check_allowed).st_uid + 1 + ) + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() diff --git a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py index 8d3962c5..531e6ffa 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py @@ -1,22 +1,73 @@ """Unit tests for startup update check.""" +import io +import json +import os +import stat import sys import threading import time -from pathlib import Path from unittest.mock import MagicMock, patch import pytest +from deepctl_cmd_update import check_cache +from deepctl_cmd_update import startup_check as mod from deepctl_cmd_update.startup_check import ( _is_ci, _is_oneshot, - _read_cache_timestamp, - _write_cache_timestamp, check_and_notify, print_pending_notification, ) +_REAL_IS_CI = mod._is_ci +_REAL_IS_AGENT = mod._is_agent + + +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture +def check_allowed(monkeypatch, cache_dir): + """A run where the startup checks may start: an interactive person at a + terminal, outside CI and ``dg mcp``. Returns the cache directory.""" + import deepctl_cmd_update.startup_check as core + + monkeypatch.delenv(core.NO_UPDATE_CHECK_ENV_VAR, raising=False) + monkeypatch.setattr(core, "_is_ci", lambda: False) + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: True) + monkeypatch.setattr(core, "_is_agent", lambda: False) + monkeypatch.setattr(core, "_is_oneshot", lambda: False) + monkeypatch.setattr(core, "_subcommand", lambda: None) + monkeypatch.setattr(core, "_get_check_interval_seconds", lambda quiet: 86400.0) + return cache_dir + + +@pytest.fixture(autouse=True) +def _reset_state(): + mod._thread = None + mod._result = {} + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + mod._result = {} + + +def _finished_thread() -> threading.Thread: + thread = threading.Thread(target=lambda: None) + thread.start() + thread.join() + return thread + + class TestCIDetection: """Test CI environment detection.""" @@ -48,14 +99,8 @@ def test_travis(self): class TestOneshotDetection: """Test one-shot execution detection.""" - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": ""}, - clear=False, - ) - def test_not_oneshot(self): - import os - os.environ.pop("UV_INTERNAL__PARENT_INTERPRETER", None) + def test_not_oneshot(self, monkeypatch): + monkeypatch.delenv("UV_INTERNAL__PARENT_INTERPRETER", raising=False) assert _is_oneshot() is False @patch.dict( @@ -67,155 +112,340 @@ def test_uvx_oneshot(self): assert _is_oneshot() is True -class TestCacheTimestamp: - """Test file-based timestamp caching.""" - - def test_read_missing_cache(self, tmp_path): - """Missing cache file returns 0.0.""" - with patch( - "deepctl_cmd_update.startup_check._CACHE_FILE", - tmp_path / "nonexistent", - ): - assert _read_cache_timestamp() == 0.0 - - def test_write_and_read_cache(self, tmp_path): - """Written timestamp can be read back.""" - cache_file = tmp_path / "last_version_check" - with patch( - "deepctl_cmd_update.startup_check._CACHE_FILE", cache_file - ), patch( - "deepctl_cmd_update.startup_check._CACHE_DIR", tmp_path - ): - _write_cache_timestamp() - ts = _read_cache_timestamp() - assert ts > 0.0 - assert abs(ts - time.time()) < 5.0 - - -class TestCheckAndNotify: - """Test the startup check lifecycle.""" - - @patch.dict("os.environ", {"CI": "true"}, clear=False) - def test_suppressed_in_ci(self): - """No thread is started in CI.""" - import deepctl_cmd_update.startup_check as mod +class TestSkipConditions: + """No check thread starts, and no file is written, when a run skips it.""" - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0") + def _assert_skipped(self, cache_dir): + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() assert mod._thread is None + assert not cache_dir.exists() - @patch( - "deepctl_cmd_update.startup_check._is_ci", return_value=False - ) - @patch( - "deepctl_cmd_update.startup_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.startup_check._get_check_interval_seconds", - return_value=None, - ) - def test_suppressed_when_never(self, *_mocks): - """No thread is started when check_frequency=never.""" - import deepctl_cmd_update.startup_check as mod + def test_starts_when_allowed(self, check_allowed): + with patch.object(mod, "_background_check", lambda ver, res: None): + check_and_notify(current_version="1.0.0") + assert mod._thread is not None + assert mod._thread.daemon is True + assert mod._thread.name == mod.THREAD_NAME + + def test_quiet(self, check_allowed): + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0", quiet=True) + thread_cls.assert_not_called() + + def test_ci(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_ci", lambda: True) + self._assert_skipped(check_allowed) + + def test_ci_env(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_ci", _REAL_IS_CI) + monkeypatch.setenv("CI", "1") + self._assert_skipped(check_allowed) + + def test_agent(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_agent", lambda: True) + self._assert_skipped(check_allowed) + + def test_ai_agent_marker(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_agent", _REAL_IS_AGENT) + monkeypatch.setenv("AI_AGENT", "x") + self._assert_skipped(check_allowed) + + def test_stderr_not_a_terminal(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_stderr_is_terminal", lambda: False) + self._assert_skipped(check_allowed) + + def test_mcp(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_subcommand", lambda: "mcp") + self._assert_skipped(check_allowed) + + @pytest.mark.parametrize("value", ["1", "true", "yes"]) + def test_opt_out_env(self, check_allowed, monkeypatch, value): + monkeypatch.setenv(mod.NO_UPDATE_CHECK_ENV_VAR, value) + self._assert_skipped(check_allowed) + + @pytest.mark.parametrize("value", ["", "0", "false"]) + def test_opt_out_env_off(self, check_allowed, monkeypatch, value): + monkeypatch.setenv(mod.NO_UPDATE_CHECK_ENV_VAR, value) + with patch.object(mod, "_background_check", lambda ver, res: None): + check_and_notify(current_version="1.0.0") + assert mod._thread is not None - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0") - assert mod._thread is None + def test_oneshot(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_oneshot", lambda: True) + self._assert_skipped(check_allowed) - def test_suppressed_in_quiet_mode(self): - """No thread is started in quiet mode.""" - import deepctl_cmd_update.startup_check as mod + def test_unusable_cache_dir(self, check_allowed, monkeypatch): + """A foreign-owned cache dir: no check thread, no network, no state.""" + check_allowed.mkdir(parents=True, mode=0o755) + monkeypatch.setattr( + check_cache.os, "getuid", lambda: os.stat(check_allowed).st_uid + 1 + ) + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + assert list(check_allowed.iterdir()) == [] - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0", quiet=True) - assert mod._thread is None + def test_never(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_get_check_interval_seconds", lambda quiet: None) + self._assert_skipped(check_allowed) - @patch.dict("os.environ", {}, clear=False) - @patch( - "deepctl_cmd_update.startup_check._is_ci", return_value=False - ) - @patch( - "deepctl_cmd_update.startup_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.startup_check._get_check_interval_seconds", - return_value=86400.0, - ) - @patch( - "deepctl_cmd_update.startup_check._read_cache_timestamp", - return_value=0.0, - ) - @patch( - "deepctl_cmd_update.startup_check._background_check", + +class TestSubcommand: + @pytest.mark.parametrize( + ("args", "expected"), + [ + (["mcp"], "mcp"), + (["-q", "mcp", "--transport", "http"], "mcp"), + (["--profile", "mcp", "models"], "models"), + (["-o", "json", "mcp"], "mcp"), + (["models", "mcp"], "models"), + (["--quiet"], None), + ], ) - def test_thread_starts( - self, - mock_bg_check, - *_mocks, + def test_reads_the_run_args(self, monkeypatch, args, expected): + from deepctl import _cli + + monkeypatch.setattr(_cli, "_run_args", args) + assert mod._subcommand() == expected + + +class TestBackoff: + """The cache file records every attempt, success or not.""" + + def test_failure_is_recorded_and_waits_a_day(self, check_allowed): + with patch("httpx.Client.get", side_effect=OSError("unreachable")): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + + state = json.loads((check_allowed / check_cache.UPDATE_FILE).read_text()) + assert state["ok"] is False + assert state["latest"] is None + assert state["source"] == "pypi" + assert abs(state["checked_at"] - time.time()) < 60 + assert mod._result == {} + + # The next run within the interval makes no attempt. + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + + def test_attempt_recorded_before_connecting(self, check_allowed): + """A check the process never lets finish still counts.""" + seen = {} + + def hang(*args, **kwargs): + path = check_allowed / check_cache.UPDATE_FILE + seen["state"] = json.loads(path.read_text()) + raise OSError("timed out") + + with patch("httpx.Client.get", side_effect=hang): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert seen["state"]["ok"] is False + + def test_nothing_recorded_until_the_client_is_built(self, check_allowed): + """Building the client (CA bundle, proxies) comes before the record.""" + with patch("httpx.Client.__init__", side_effect=OSError("no CA bundle")): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert not (check_allowed / check_cache.UPDATE_FILE).exists() + + def test_run_ending_before_the_request_keeps_the_check_due( + self, check_allowed, monkeypatch ): - """A daemon thread is started when conditions are met.""" - import deepctl_cmd_update.startup_check as mod - - mod._thread = None - mod._result = {} - - # Make _background_check a no-op - mock_bg_check.side_effect = lambda ver, res: None - + """Nothing is recorded until the request is about to go out.""" + monkeypatch.setitem(sys.modules, "httpx", None) # import fails check_and_notify(current_version="1.0.0") - assert mod._thread is not None - assert mod._thread.daemon is True - mod._thread.join(timeout=2.0) + mod._thread.join(timeout=5.0) + assert not (check_allowed / check_cache.UPDATE_FILE).exists() + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + def test_success_is_recorded(self, check_allowed): + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with patch("httpx.Client.get", return_value=response): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + + state = json.loads((check_allowed / check_cache.UPDATE_FILE).read_text()) + assert state["ok"] is True + assert state["latest"] == "2.0.0" + assert mod._result == {"latest": "2.0.0", "current": "1.0.0"} + + def test_due_again_after_the_interval(self, check_allowed): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + path = check_allowed / check_cache.UPDATE_FILE + state = json.loads(path.read_text()) + state["checked_at"] -= 86401 + path.write_text(json.dumps(state)) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + def test_future_timestamp_is_due(self, check_allowed): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + path = check_allowed / check_cache.UPDATE_FILE + state = json.loads(path.read_text()) + state["checked_at"] += 10 * 86400 + path.write_text(json.dumps(state)) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +class TestCacheFile: + def test_modes(self, cache_dir): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + path = cache_dir / check_cache.UPDATE_FILE + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert [p.name for p in cache_dir.iterdir()] == [check_cache.UPDATE_FILE] + + def test_created_private_under_umask_002(self, cache_dir): + old = os.umask(0o002) + try: + assert check_cache.usable() + finally: + os.umask(old) + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + + @pytest.mark.parametrize("mode", [0o775, 0o770, 0o755, 0o777]) + def test_own_permissive_dir_is_tightened(self, cache_dir, mode): + """Older versions made this directory with a plain mkdir (0775 under umask 002).""" + cache_dir.mkdir(parents=True) + os.chmod(cache_dir, mode) + assert check_cache.usable() + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert check_cache.read(check_cache.UPDATE_FILE) is not None + + def test_foreign_owned_dir_is_unusable(self, cache_dir, monkeypatch): + cache_dir.mkdir(parents=True, mode=0o755) + monkeypatch.setattr(check_cache.os, "getuid", lambda: os.stat(cache_dir).st_uid + 1) + assert check_cache.usable() is False + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o755 # left alone + + def test_tightening_failure_is_unusable(self, cache_dir, monkeypatch): + cache_dir.mkdir(parents=True) + os.chmod(cache_dir, 0o775) + + def refuse(fd, mode): + raise PermissionError("read-only") + + monkeypatch.setattr(check_cache.os, "fchmod", refuse) + assert check_cache.usable() is False + + def test_symlinked_dir_is_unusable(self, cache_dir, tmp_path): + other = tmp_path / "other" + other.mkdir(mode=0o700) + cache_dir.parent.mkdir(parents=True) + cache_dir.symlink_to(other) + assert check_cache.usable() is False + + def test_group_writable_dir_is_never_written_without_tightening(self, cache_dir): + cache_dir.mkdir(parents=True, mode=0o700) + os.chmod(cache_dir, 0o770) + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert list(cache_dir.iterdir()) == [] + assert check_cache.read(check_cache.UPDATE_FILE) is None + + def test_group_writable_file_is_not_read(self, cache_dir): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + os.chmod(cache_dir / check_cache.UPDATE_FILE, 0o666) + assert check_cache.read(check_cache.UPDATE_FILE) is None + + def test_symlinked_file_is_not_read(self, cache_dir, tmp_path): + cache_dir.mkdir(parents=True, mode=0o700) + target = tmp_path / "elsewhere.json" + target.write_text(json.dumps({"checked_at": time.time()})) + os.chmod(target, 0o600) + (cache_dir / check_cache.UPDATE_FILE).symlink_to(target) + assert check_cache.read(check_cache.UPDATE_FILE) is None + + @pytest.mark.parametrize("data", ["not json", "[]", '{"checked_at": true}']) + def test_bad_contents_read_as_never_checked(self, cache_dir, data): + cache_dir.mkdir(parents=True, mode=0o700) + path = cache_dir / check_cache.UPDATE_FILE + path.write_text(data) + os.chmod(path, 0o600) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + +class TestNoConfigWrite: + def test_check_never_saves_the_config(self, check_allowed): + from deepctl_core import Config + + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with ( + patch.object(Config, "save") as save, + patch.object(Config, "_set_config_value") as set_value, + patch("httpx.Client.get", return_value=response), + ): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + save.assert_not_called() + set_value.assert_not_called() class TestPrintPendingNotification: """Test notification display.""" def test_no_notification_when_no_thread(self, capsys): - """No output when no thread was started.""" - import deepctl_cmd_update.startup_check as mod - - mod._thread = None - mod._result = {} print_pending_notification() captured = capsys.readouterr() assert captured.err == "" def test_notification_displayed(self): """Notification is printed to stderr when update found.""" - import deepctl_cmd_update.startup_check as mod - - # Simulate completed thread with result mod._result = {"latest": "2.0.0", "current": "1.0.0"} - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() - - # Capture stderr - import io + mod._thread = _finished_thread() stderr_capture = io.StringIO() with patch.object(sys, "stderr", stderr_capture): print_pending_notification() output = stderr_capture.getvalue() + assert output.count("Update available") == 1 assert "2.0.0" in output assert "1.0.0" in output assert "dg update" in output + # Printed once: a second call has nothing to show. + stderr_capture = io.StringIO() + with patch.object(sys, "stderr", stderr_capture): + print_pending_notification() + assert stderr_capture.getvalue() == "" + + def test_running_check_is_dropped_without_waiting(self): + """A check still running at exit is never joined.""" + release = threading.Event() + mod._result = {"latest": "2.0.0", "current": "1.0.0"} + mod._thread = threading.Thread(target=release.wait, daemon=True) + mod._thread.start() + thread = mod._thread + try: + with patch.object(thread, "join") as join: + stderr_capture = io.StringIO() + started = time.monotonic() + with patch.object(sys, "stderr", stderr_capture): + print_pending_notification() + elapsed = time.monotonic() - started + join.assert_not_called() + assert stderr_capture.getvalue() == "" + assert elapsed < 2.0 + assert mod._thread is None + finally: + release.set() + thread.join(timeout=5.0) + def test_no_notification_when_up_to_date(self, capsys): """No output when no update is available.""" - import deepctl_cmd_update.startup_check as mod - mod._result = {} # No update found - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() + mod._thread = _finished_thread() print_pending_notification() captured = capsys.readouterr() @@ -230,15 +460,99 @@ def test_no_notification_when_up_to_date(self, capsys): ) def test_broken_pipe_swallowed(self, error): """A closed/broken stderr (e.g. `dg mcp` host disconnect) is tolerated.""" - import deepctl_cmd_update.startup_check as mod - mod._result = {"latest": "2.0.0", "current": "1.0.0"} - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() + mod._thread = _finished_thread() broken_stderr = MagicMock() broken_stderr.write.side_effect = error with patch.object(sys, "stderr", broken_stderr): # Must not raise. print_pending_notification() + + +class TestSavedNotice: + """An answer that came too late for its run gives the notice later.""" + + @staticmethod + def _save(cache_dir, **state): + assert check_cache.usable() + path = cache_dir / check_cache.UPDATE_FILE + path.write_text(json.dumps(state)) + os.chmod(path, 0o600) + + @staticmethod + def _notice() -> str: + err = io.StringIO() + with patch.object(sys, "stderr", err): + print_pending_notification() + return err.getvalue() + + def test_printed_once_per_interval(self, check_allowed): + self._save(check_allowed, checked_at=time.time(), ok=True, latest="2.0.0") + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + out = self._notice() + assert out.count("Update available") == 1 and "2.0.0" in out + state = check_cache.read(check_cache.UPDATE_FILE) + assert abs(check_cache.notified_at(state) - time.time()) < 60 + + check_and_notify(current_version="1.0.0") + assert self._notice() == "" + + def test_again_after_the_interval(self, check_allowed): + self._save( + check_allowed, + checked_at=time.time(), + ok=True, + latest="2.0.0", + notified_at=time.time() - 86401, + ) + check_and_notify(current_version="1.0.0") + assert "2.0.0" in self._notice() + + @pytest.mark.parametrize( + "state", + [ + {"ok": True, "latest": "1.0.0"}, + {"ok": True, "latest": "0.9.0"}, + {"ok": False, "latest": None}, + {"ok": True, "latest": "not a version"}, + ], + ) + def test_nothing_newer_no_notice(self, check_allowed, state): + self._save(check_allowed, checked_at=time.time(), **state) + check_and_notify(current_version="1.0.0") + assert self._notice() == "" + + def test_late_answer_reaches_the_next_run(self, check_allowed): + """A sub-second command drops the result; the second run shows it.""" + release = threading.Event() + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + + def slow_get(*args, **kwargs): + release.wait(5.0) + return response + + with patch("httpx.Client.get", side_effect=slow_get): + check_and_notify(current_version="1.0.0") + thread = mod._thread + assert self._notice() == "" # run 1 ends before the answer + release.set() + thread.join(timeout=5.0) + + check_and_notify(current_version="1.0.0") # run 2: not due + assert mod._thread is None + out = self._notice() + assert out.count("Update available") == 1 and "1.0.0 → 2.0.0" in out + + def test_in_time_notice_is_not_repeated_from_the_cache(self, check_allowed): + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with patch("httpx.Client.get", return_value=response): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert "2.0.0" in self._notice() + check_and_notify(current_version="1.0.0") + assert self._notice() == "" diff --git a/packages/deepctl-cmd-update/tests/unit/test_update_command.py b/packages/deepctl-cmd-update/tests/unit/test_update_command.py index 3a89f6da..8e9c8246 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_update_command.py +++ b/packages/deepctl-cmd-update/tests/unit/test_update_command.py @@ -1,5 +1,6 @@ """Unit tests for update command.""" +import os import subprocess from unittest.mock import MagicMock, patch @@ -9,6 +10,55 @@ from deepctl_cmd_update.version_check import VersionInfo +def _real_config(tmp_path, monkeypatch): + """The default user config (so update.* keys are honored), in tmp_path.""" + from deepctl_core import Config + + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + path = tmp_path / "xdg" / "deepctl" / "config.yaml" + path.parent.mkdir(parents=True, mode=0o700) + path.write_text( + "profiles:\n default:\n project_id: P1\n" + "update:\n installation_method: pipx\n check_frequency: daily\n" + ) + os.chmod(path, 0o600) + config = Config() + assert config.config_path == path + return config, path + + +class TestConfigUntouched: + """dg update never writes the user config (no implicit save).""" + + @pytest.mark.parametrize("returncode", [0, 1]) + @patch("deepctl_cmd_update.command.subprocess.run") + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + @patch("deepctl_cmd_update.command.asyncio.run") + def test_full_update_leaves_the_file_byte_identical( + self, run_check, detect, run, returncode, tmp_path, monkeypatch + ): + config, path = _real_config(tmp_path, monkeypatch) + before = (path.read_bytes(), path.stat().st_mtime_ns) + run_check.return_value = VersionInfo( + current_version="0.1.0", latest_version="0.2.0", update_available=True + ) + detect.return_value = InstallationInfo( + method=InstallMethod.PIP, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + run.return_value = subprocess.CompletedProcess([], returncode, "", "") + UpdateCommand().handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), yes=True + ) + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + # The legacy key is still readable, and was not rewritten. + assert config.get("update.installation_method") == "pipx" + + class TestUpdateCommand: """Test update command functionality.""" diff --git a/packages/deepctl-cmd-update/tests/unit/test_version_check.py b/packages/deepctl-cmd-update/tests/unit/test_version_check.py index f96f4fc6..bde34013 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_version_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_version_check.py @@ -14,6 +14,25 @@ ) +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture(autouse=True) +def _isolated_cache(cache_dir): + """Every test here gets its own empty cache directory.""" + from deepctl_cmd_update import check_cache + + assert check_cache.usable() + return cache_dir + + class TestCheckFrequency: """Test check_frequency configuration values.""" @@ -274,3 +293,93 @@ def test_update_available(self): assert "1.0.0" in msg assert "2.0.0" in msg assert "dg update" in msg + + +class TestCacheFile: + """The last answer lives in the cache file, never in the user config.""" + + @staticmethod + def _config(values=None): + config = MagicMock() + values = {"update.check_enabled": True, **(values or {})} + config.get.side_effect = lambda key, default=None: values.get(key, default) + return config + + @pytest.mark.asyncio + async def test_answer_goes_to_the_cache_file(self, cache_dir): + from deepctl_cmd_update import check_cache + + config = self._config() + checker = VersionChecker(config, "1.0.0") + info = VersionInfo( + current_version="1.0.0", latest_version="2.0.0", update_available=True + ) + with patch.object(checker, "_check_pypi", AsyncMock(return_value=info)): + await checker.check_version(force=True) + + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None + assert (state["ok"], state["latest"], state["source"]) == ( + True, + "2.0.0", + "pypi", + ) + config.save.assert_not_called() + config._set_config_value.assert_not_called() + + @pytest.mark.asyncio + async def test_github_answer_names_its_source(self): + from deepctl_cmd_update import check_cache + + checker = VersionChecker(self._config(), "1.0.0") + response = MagicMock() + response.json.return_value = {"tag_name": "v2.0.0", "html_url": "u"} + client = AsyncMock() + client.get.return_value = response + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + with patch("httpx.AsyncClient", return_value=client): + await checker.check_version(force=True, install_method="homebrew") + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None and state["source"] == "github" + + def test_cache_file_wins_over_legacy_keys(self): + from deepctl_cmd_update import check_cache + + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="3.0.0", source="pypi") + config = self._config( + { + "update.last_check": "2000-01-01T00:00:00+00:00", + "update.cached_version_info": { + "current_version": "1.0.0", + "latest_version": "9.9.9", + "update_available": True, + }, + } + ) + checker = VersionChecker(config, "1.0.0") + assert checker.should_check() is False + info = checker._get_cached_info() + assert info is not None + assert (info.latest_version, info.update_available) == ("3.0.0", True) + + def test_failed_check_has_no_cached_answer(self): + from deepctl_cmd_update import check_cache + + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + checker = VersionChecker(self._config(), "1.0.0") + assert checker.should_check() is False + assert checker._get_cached_info() is None + + def test_legacy_keys_read_when_no_cache_file(self): + config = self._config( + { + "update.cached_version_info": { + "current_version": "1.0.0", + "latest_version": "1.5.0", + "update_available": True, + }, + } + ) + info = VersionChecker(config, "1.0.0")._get_cached_info() + assert info is not None and info.latest_version == "1.5.0" diff --git a/tests/unit/test_base_url_persistence.py b/tests/unit/test_base_url_persistence.py index 88f00da6..3cd0479a 100644 --- a/tests/unit/test_base_url_persistence.py +++ b/tests/unit/test_base_url_persistence.py @@ -90,14 +90,6 @@ def _ffprobe_reset(config, auth, client): FfprobeCommand().handle(config, auth, client, reset=True) -def _version_cache(config, auth, client): - VersionChecker(config)._cache_info( - VersionInfo( - current_version="0.4.0", latest_version="0.4.1", update_available=True - ) - ) - - def _device_login(config, auth, client): auth._store_token( TokenResponse( @@ -124,7 +116,6 @@ def _delete_and_create(config, auth, client): _switch_profile, _ffprobe_set, _ffprobe_reset, - _version_cache, _device_login, _update_install_method, _delete_and_create, @@ -171,3 +162,21 @@ def test_command_never_saves_runtime_base_url(tmp_path, monkeypatch, saver, via) monkeypatch.delenv("DEEPGRAM_BASE_URL", raising=False) later = Config(config_path=str(path)) assert later.get_profile().base_url == "https://api.deepgram.com" + + +def test_version_cache_never_writes_the_config(tmp_path, monkeypatch): + """The update check's answer goes to the cache directory instead.""" + from deepctl import _cache + + monkeypatch.setattr(_cache, "cache_root", lambda: str(tmp_path / "cache")) + path = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump({"profiles": {"default": {"project_id": "P1"}}})) + before = path.read_bytes() + config = Config(config_path=str(path), base_url=URL) + VersionChecker(config)._cache_info( + VersionInfo( + current_version="0.4.0", latest_version="0.4.1", update_available=True + ) + ) + assert path.read_bytes() == before + assert (tmp_path / "cache" / "update-check.json").is_file() From d30a717196bc7b202f9f5072624e2644caa836f9 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 13:15:35 -0500 Subject: [PATCH 7/8] fix(update): detect pipx, uv and npm installs correctly Install detection now looks at sys.prefix, not the interpreter it links to. A pipx or uv tool environment on Homebrew's Python resolves its interpreter into the Cellar, which used to read as a Homebrew install and send dg update to brew upgrade. - Homebrew: sys.prefix under Cellar/deepgram/ only. - uvx: sys.prefix in uv's cache under archive-v*, honoring UV_CACHE_DIR. The uv internal interpreter variable alone no longer counts, since uv run sets it for project environments too. - pipx and pipx run: sys.prefix under pipx's venvs or .cache; a PIPX_HOME variable alone no longer counts. - uv tool: sys.prefix directly under the uv tool directory; dg update runs uv tool install deepctl@latest, since uv tool upgrade leaves a pinned install where it is. - npm: DEEPCTL_LAUNCHER=npm or npx; dg update prints npm i -g @deepgram/cli@latest. The startup check skips npx runs as it does uvx and pipx run, and the uvx advice now covers a uv tool install shadowing uvx. dg update --dry-run prints the detected method and the command that updates it, without checking or installing. --- .../src/deepctl_cmd_update/channel.py | 188 +++++++++++ .../src/deepctl_cmd_update/command.py | 41 +++ .../src/deepctl_cmd_update/installation.py | 91 +++--- .../src/deepctl_cmd_update/startup_check.py | 15 +- .../tests/unit/test_installation.py | 307 +++++++++++------- .../tests/unit/test_startup_check.py | 25 +- .../tests/unit/test_update_command.py | 82 +++++ src/deepctl/_manifest.d/update.json | 23 +- 8 files changed, 577 insertions(+), 195 deletions(-) create mode 100644 packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py new file mode 100644 index 00000000..3b21004f --- /dev/null +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py @@ -0,0 +1,188 @@ +"""Which tool installed this copy of deepctl, judged from its own layout. + +Stdlib only: the startup check uses these on every run. + +Each test looks at ``sys.prefix``, the environment deepctl runs in, not at +the interpreter it links to. A pipx or ``uv tool`` environment built on +Homebrew's Python resolves its interpreter into Homebrew's Cellar, so the +interpreter can't tell those installs from Homebrew's own ``deepgram`` +formula. +""" + +from __future__ import annotations + +import os +import re +import sys +from itertools import pairwise +from pathlib import Path + +LAUNCHER_ENV_VAR = "DEEPCTL_LAUNCHER" +NPM_LAUNCHERS = ("npm", "npx") + +_UV_ARCHIVE = re.compile(r"archive-v\d+") + + +def launcher() -> str | None: + """``npm`` or ``npx`` when the @deepgram/cli npm package started dg. + + ``deepctl.main`` removes ``DEEPCTL_LAUNCHER`` from the environment at + startup and keeps its value; the environment is the fallback when dg + didn't start through ``deepctl.main``. + """ + main = sys.modules.get("deepctl.main") + if main is not None and getattr(main, "LAUNCHER_CAPTURED", False): + value = getattr(main, "LAUNCHER", None) + else: + value = os.environ.get(LAUNCHER_ENV_VAR) + value = (value or "").strip().lower() + return value if value in NPM_LAUNCHERS else None + + +def _home() -> Path | None: + try: + return Path.home() + except (KeyError, RuntimeError): + return None + + +def _env_path(name: str) -> Path | None: + value = os.environ.get(name) + return Path(value) if value else None + + +def _prefixes() -> list[Path]: + """``sys.prefix`` as given and with links resolved.""" + prefix = Path(sys.prefix) + paths = [prefix] + try: + resolved = prefix.resolve() + except OSError: + return paths + if resolved != prefix: + paths.append(resolved) + return paths + + +def _parts_below(base: Path) -> tuple[str, ...] | None: + """The parts of ``sys.prefix`` below ``base``, or ``None`` if it isn't + inside ``base`` (each compared as given and with links resolved).""" + bases = [base] + try: + resolved = base.resolve() + except OSError: + resolved = base + if resolved != base: + bases.append(resolved) + for prefix in _prefixes(): + for candidate in bases: + try: + return prefix.relative_to(candidate).parts + except ValueError: + continue + return None + + +def _first(paths: list[Path | None]) -> list[Path]: + """The given paths without ``None`` and duplicates, in order.""" + seen: list[Path] = [] + for path in paths: + if path is not None and path not in seen: + seen.append(path) + return seen + + +def is_brew() -> bool: + """Homebrew's ``deepgram`` formula: ``sys.prefix`` under + ``Cellar/deepgram/``.""" + for prefix in _prefixes(): + parts = prefix.parts + if any(pair == ("Cellar", "deepgram") for pair in pairwise(parts)): + return True + return False + + +def uv_cache_dirs() -> list[Path]: + """Where uv keeps its cache: ``UV_CACHE_DIR``, else its defaults.""" + home = _home() + xdg = _env_path("XDG_CACHE_HOME") + local = _env_path("LOCALAPPDATA") + return _first( + [ + _env_path("UV_CACHE_DIR"), + xdg / "uv" if xdg else None, + home / ".cache" / "uv" if home else None, + local / "uv" / "cache" if local and sys.platform == "win32" else None, + ] + ) + + +def is_uvx() -> bool: + """``uvx deepctl``: an environment in uv's cache (``archive-v*``).""" + for cache in uv_cache_dirs(): + parts = _parts_below(cache) + if parts and _UV_ARCHIVE.fullmatch(parts[0]): + return True + return False + + +def uv_tool_dirs() -> list[Path]: + """Where ``uv tool install`` puts tools: ``UV_TOOL_DIR``, else its + defaults.""" + home = _home() + xdg = _env_path("XDG_DATA_HOME") + appdata = _env_path("APPDATA") + return _first( + [ + _env_path("UV_TOOL_DIR"), + xdg / "uv" / "tools" if xdg else None, + home / ".local" / "share" / "uv" / "tools" if home else None, + appdata / "uv" / "data" / "tools" + if appdata and sys.platform == "win32" + else None, + ] + ) + + +def is_uv_tool() -> bool: + """``uv tool install deepctl``: ``sys.prefix`` is ``/``.""" + return any(len(_parts_below(d) or ()) == 1 for d in uv_tool_dirs()) + + +def pipx_homes() -> list[Path]: + """pipx's home: ``PIPX_HOME``, else the locations pipx versions use.""" + home = _home() + xdg = _env_path("XDG_DATA_HOME") + return _first( + [ + _env_path("PIPX_HOME"), + xdg / "pipx" if xdg else None, + home / ".local" / "share" / "pipx" if home else None, + home / ".local" / "pipx" if home else None, + home / "Library" / "Application Support" / "pipx" if home else None, + home / "pipx" if home else None, + ] + ) + + +def _in_pipx(area: str) -> bool: + return any( + len(parts) == 2 and parts[0] == area + for parts in (_parts_below(home) for home in pipx_homes()) + if parts + ) + + +def is_pipx() -> bool: + """``pipx install deepctl``: ``sys.prefix`` is ``/venvs/``.""" + return _in_pipx("venvs") + + +def is_pipx_run() -> bool: + """``pipx run deepctl``: ``sys.prefix`` is ``/.cache/``.""" + return _in_pipx(".cache") + + +def is_oneshot() -> bool: + """A run that fetches deepctl for one use: uvx, pipx run or npx.""" + return is_uvx() or is_pipx_run() or launcher() == "npx" diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py index 675393ec..4ced9a69 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py @@ -14,6 +14,7 @@ print_success, print_warning, ) +from rich.markup import escape from rich.panel import Panel from rich.prompt import Confirm @@ -42,6 +43,7 @@ class UpdateCommand(BaseCommand): examples = [ "dg update", "dg update --check-only", + "dg update --dry-run", ] agent_help = ( "Check for and install updates to deepctl. Can check PyPI for newer " @@ -58,6 +60,15 @@ def get_arguments(self) -> list[dict[str, Any]]: "is_flag": True, "is_option": True, }, + { + "names": ["--dry-run"], + "help": ( + "Show how deepctl was installed and the command that " + "updates it, without checking for or installing updates" + ), + "is_flag": True, + "is_option": True, + }, { "names": ["--force"], "help": "Force update even if already up to date", @@ -94,6 +105,8 @@ def handle( # Extract arguments from kwargs check_only = kwargs.get("check_only", False) + if kwargs.get("dry_run", False): + return self._dry_run() force = kwargs.get("force", False) yes = kwargs.get("yes", False) @@ -254,3 +267,31 @@ def handle( update_available=version_info.update_available, installation_method=install_info.method.value, ) + + def _dry_run(self) -> dict[str, Any]: + """Print the installation method and its update command; change + nothing and make no network request. + + The lines print only in the default format: in the others, stdout + carries just the payload, whose ``message`` holds the same command. + """ + from deepctl_core.output import get_output_format + + detector = InstallationDetector() + install_info = detector.detect() + update_command = detector.get_update_command(install_info.method) + if update_command: + message = f"Run: {shlex.join(update_command)}" + else: + message = detector.get_update_instructions(install_info) + if get_output_format() == "default": + console = get_console() + method = install_info.method.value + console.print(f"Installation method: [cyan]{method}[/cyan]") + console.print(f"Installation path: [dim]{escape(install_info.path)}[/dim]") + console.print(message, markup=False, highlight=False) + return UpdateResult( + success=True, + message=message, + installation_method=install_info.method.value, + ).model_dump() diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py index 6fda2fe9..c11345d6 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py @@ -9,6 +9,10 @@ from pydantic import BaseModel +from . import channel + +NPM_UPDATE_COMMAND = "npm i -g @deepgram/cli@latest" + class InstallMethod(str, Enum): """Supported installation methods.""" @@ -20,6 +24,7 @@ class InstallMethod(str, Enum): UV_TOOL = "uv_tool" UVX = "uvx" PIPX_RUN = "pipx_run" + NPM = "npm" SYSTEM = "system" DEVELOPMENT = "development" UNKNOWN = "unknown" @@ -97,14 +102,16 @@ def get_update_command(self, method: InstallMethod) -> list[str] | None: "deepctl", ], InstallMethod.HOMEBREW: ["brew", "upgrade", "deepgram"], + # `uv tool upgrade` leaves a pinned install where it is. InstallMethod.UV_TOOL: [ "uv", "tool", - "upgrade", - "deepctl", + "install", + "deepctl@latest", ], InstallMethod.UVX: None, # Ephemeral, no update needed InstallMethod.PIPX_RUN: None, # Ephemeral, no update needed + InstallMethod.NPM: None, # Printed, not run: npm owns the install InstallMethod.SYSTEM: None, # Handled separately InstallMethod.DEVELOPMENT: None, # Can't auto-update InstallMethod.UNKNOWN: None, @@ -125,10 +132,22 @@ def get_update_instructions(self, info: InstallationInfo) -> str: if info.method == InstallMethod.UVX: return ( - "You are running deepctl via uvx (one-shot execution). " - "The latest version will be used automatically on next run." + "uvx runs the latest deepctl each time, so there's nothing to " + "update. If you also installed deepctl with uv tool install, " + "uvx runs that copy instead; update it with: " + "uv tool install deepctl@latest. " + "To pin a version, run uvx deepctl@." ) + if info.method == InstallMethod.NPM: + if channel.launcher() == "npx": + return ( + "npx runs the latest @deepgram/cli each time, so there's " + "nothing to update. To pin a version, run " + "npx @deepgram/cli@." + ) + return f"Run: {NPM_UPDATE_COMMAND}" + if info.method == InstallMethod.PIPX_RUN: return ( "You are running deepctl via pipx run (one-shot execution). " @@ -263,8 +282,8 @@ def _detect_method( ) -> InstallMethod: """Detect the installation method. - Priority: development > oneshot > homebrew > uv_tool > pipx > - uv > system > pip > unknown + Priority: development > npm > oneshot > homebrew > uv_tool > + pipx > uv > system > pip > unknown Args: package_location: Path to package @@ -278,6 +297,10 @@ def _detect_method( if is_editable: return InstallMethod.DEVELOPMENT + # Started by the @deepgram/cli npm package (npm or npx) + if channel.launcher() is not None: + return InstallMethod.NPM + # One-shot execution (uvx / pipx run) oneshot = self._detect_oneshot_execution() if oneshot is not None: @@ -313,24 +336,16 @@ def _detect_method( return InstallMethod.UNKNOWN def _is_homebrew_install(self) -> bool: - """Check if installed via Homebrew. + """Check if installed by Homebrew's ``deepgram`` formula. - Detects Apple Silicon (/opt/homebrew), Intel Mac - (/usr/local/Cellar), and Linux Homebrew paths. + Only ``sys.prefix`` under ``Cellar/deepgram/`` counts: a pipx or + uv tool environment on Homebrew's Python resolves its interpreter + into the Cellar too, and isn't a Homebrew install of deepctl. Returns: True if Homebrew installation """ - exe_path = Path(sys.executable).resolve() - exe_str = str(exe_path) - - homebrew_prefixes = [ - "/opt/homebrew", # Apple Silicon - "/usr/local/Cellar", # Intel Mac - "/home/linuxbrew/.linuxbrew", # Linux - ] - - return any(exe_str.startswith(prefix) for prefix in homebrew_prefixes) + return channel.is_brew() def _is_uv_tool_install(self) -> bool: """Check if installed via ``uv tool install``. @@ -338,24 +353,7 @@ def _is_uv_tool_install(self) -> bool: Returns: True if uv tool installation """ - # Use sys.prefix (the venv root), not sys.executable — uv's Python - # interpreter is a symlink to a shared binary pool outside tools/, - # so resolve() on the executable breaks the path check. - prefix_str = str(Path(sys.prefix)) - exe_str = str(Path(sys.executable)) - - # Check custom UV_TOOL_DIR env var first - uv_tool_dir = os.environ.get("UV_TOOL_DIR") - if uv_tool_dir and ( - prefix_str.startswith(uv_tool_dir) or exe_str.startswith(uv_tool_dir) - ): - return True - - # Default uv tool location - default_uv_tools = str(Path.home() / ".local" / "share" / "uv" / "tools") - return prefix_str.startswith(default_uv_tools) or exe_str.startswith( - default_uv_tools - ) + return channel.is_uv_tool() def _detect_oneshot_execution(self) -> InstallMethod | None: """Detect ephemeral one-shot execution (uvx / pipx run). @@ -363,17 +361,10 @@ def _detect_oneshot_execution(self) -> InstallMethod | None: Returns: UVX or PIPX_RUN if detected, None otherwise """ - # uvx sets UV_INTERNAL__PARENT_INTERPRETER - if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: + if channel.is_uvx(): return InstallMethod.UVX - - # pipx run uses a cache directory - exe_path = str(Path(sys.executable).resolve()) - pipx_home = os.environ.get("PIPX_HOME", str(Path.home() / ".local" / "pipx")) - pipx_cache = str(Path(pipx_home) / ".cache") - if exe_path.startswith(pipx_cache): + if channel.is_pipx_run(): return InstallMethod.PIPX_RUN - return None def _is_pipx_install(self) -> bool: @@ -382,13 +373,7 @@ def _is_pipx_install(self) -> bool: Returns: True if pipx installation """ - # Check for pipx environment variables - if "PIPX_HOME" in os.environ or "PIPX_BIN_DIR" in os.environ: - return True - - # Check if executable is in a pipx-managed location - exe_path = Path(sys.executable) - return "pipx" in str(exe_path) or (exe_path.parent.parent / ".pipx").exists() + return channel.is_pipx() def _is_uv_install(self) -> bool: """Check if installed via uv. diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py index 9d6693db..fbd8d148 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py @@ -28,7 +28,7 @@ import threading from typing import Any -from . import check_cache +from . import channel, check_cache PYPI_URL = "https://pypi.org/pypi/deepctl/json" NO_UPDATE_CHECK_ENV_VAR = "DEEPCTL_NO_UPDATE_CHECK" @@ -54,17 +54,8 @@ def _is_ci() -> bool: def _is_oneshot() -> bool: - """Return True if running in an ephemeral one-shot environment.""" - # uvx - if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: - return True - # pipx run - from pathlib import Path - - exe_path = str(Path(sys.executable).resolve()) - pipx_home = os.environ.get("PIPX_HOME", str(Path.home() / ".local" / "pipx")) - pipx_cache = str(Path(pipx_home) / ".cache") - return bool(exe_path.startswith(pipx_cache)) + """Return True for a one-shot run (uvx, pipx run, npx): nothing to update.""" + return channel.is_oneshot() def _opted_out() -> bool: diff --git a/packages/deepctl-cmd-update/tests/unit/test_installation.py b/packages/deepctl-cmd-update/tests/unit/test_installation.py index 22cd68ff..32141586 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_installation.py +++ b/packages/deepctl-cmd-update/tests/unit/test_installation.py @@ -22,7 +22,7 @@ def test_all_methods_defined(self): """Verify all expected methods exist.""" expected = { "pip", "pipx", "uv", "homebrew", "uv_tool", - "uvx", "pipx_run", "system", "development", "unknown", + "uvx", "pipx_run", "npm", "system", "development", "unknown", } actual = {m.value for m in InstallMethod} assert actual == expected @@ -58,7 +58,7 @@ def test_returns_list_for_homebrew(self, detector): def test_returns_list_for_uv_tool(self, detector): cmd = detector.get_update_command(InstallMethod.UV_TOOL) assert isinstance(cmd, list) - assert cmd == ["uv", "tool", "upgrade", "deepctl"] + assert cmd == ["uv", "tool", "install", "deepctl@latest"] def test_returns_none_for_uvx(self, detector): cmd = detector.get_update_command(InstallMethod.UVX) @@ -77,143 +77,208 @@ def test_returns_none_for_unknown(self, detector): assert cmd is None -class TestHomebrewDetection: - """Test Homebrew installation detection.""" - - @pytest.fixture - def detector(self): +@pytest.fixture +def layout(tmp_path, monkeypatch): + """A home directory with no tool settings, and a way to set sys.prefix.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + for name in ( + "UV_CACHE_DIR", + "UV_TOOL_DIR", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "PIPX_HOME", + "PIPX_BIN_DIR", + "DEEPCTL_LAUNCHER", + "UV_INTERNAL__PARENT_INTERPRETER", + "UV_PROJECT", + ): + monkeypatch.delenv(name, raising=False) + main = sys.modules.get("deepctl.main") + if main is not None: + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", False, raising=False) + + def set_prefix(path): + monkeypatch.setattr(sys, "prefix", str(path)) return InstallationDetector() - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_apple_silicon_path(self, mock_sys): - """Detect Homebrew on Apple Silicon.""" - mock_sys.executable = "/opt/homebrew/bin/python3" - detector = InstallationDetector() - # Patch resolve to return the same path - with patch.object( - Path, "resolve", return_value=Path("/opt/homebrew/bin/python3") - ): - assert detector._is_homebrew_install() is True - - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_intel_mac_path(self, mock_sys): - """Detect Homebrew on Intel Mac.""" - mock_sys.executable = "/usr/local/Cellar/python@3.12/bin/python3" - detector = InstallationDetector() - with patch.object( - Path, - "resolve", - return_value=Path( - "/usr/local/Cellar/python@3.12/bin/python3" - ), - ): - assert detector._is_homebrew_install() is True - - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_linux_homebrew_path(self, mock_sys): - """Detect Homebrew on Linux.""" - mock_sys.executable = ( - "/home/linuxbrew/.linuxbrew/bin/python3" + set_prefix.home = home + return set_prefix + + +@unix_only +class TestHomebrewDetection: + """Homebrew means the deepgram formula's own Cellar prefix.""" + + @pytest.mark.parametrize( + "prefix", + [ + "/opt/homebrew/Cellar/deepgram/0.5.0/libexec", + "/usr/local/Cellar/deepgram/0.5.0/libexec", + "/home/linuxbrew/.linuxbrew/Cellar/deepgram/0.5.0/libexec", + ], + ) + def test_formula_prefix(self, layout, prefix): + assert layout(prefix)._is_homebrew_install() is True + + @pytest.mark.parametrize( + "prefix", + [ + "/opt/homebrew/Cellar/python@3.12/3.12.7/Frameworks/Python.framework", + "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/3.12.7", + "/usr", + ], + ) + def test_homebrew_python_alone_is_not_brew(self, layout, prefix): + assert layout(prefix)._is_homebrew_install() is False + + def test_pipx_on_homebrew_python(self, layout, monkeypatch): + """A pipx venv whose interpreter resolves into the Cellar is pipx.""" + prefix = layout.home / ".local" / "share" / "pipx" / "venvs" / "deepctl" + monkeypatch.setattr( + sys, "executable", "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/bin/python3" ) - detector = InstallationDetector() - with patch.object( - Path, - "resolve", - return_value=Path( - "/home/linuxbrew/.linuxbrew/bin/python3" - ), - ): - assert detector._is_homebrew_install() is True + detector = layout(prefix) + assert detector._is_homebrew_install() is False + assert detector._detect_method(prefix, True, False) == InstallMethod.PIPX + + def test_uv_tool_on_homebrew_python(self, layout, monkeypatch): + prefix = layout.home / ".local" / "share" / "uv" / "tools" / "deepctl" + monkeypatch.setattr( + sys, "executable", "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/bin/python3" + ) + detector = layout(prefix) + assert detector._detect_method(prefix, True, False) == InstallMethod.UV_TOOL - @patch("deepctl_cmd_update.installation.sys") - def test_non_homebrew_path(self, mock_sys): - """Non-Homebrew path returns False.""" - mock_sys.executable = "/usr/bin/python3" - detector = InstallationDetector() - with patch.object( - Path, "resolve", return_value=Path("/usr/bin/python3") - ): - assert detector._is_homebrew_install() is False + def test_formula_through_opt_link(self, layout, tmp_path): + cellar = tmp_path / "brew" / "Cellar" / "deepgram" / "0.5.0" / "libexec" + cellar.mkdir(parents=True) + opt = tmp_path / "brew" / "opt" + opt.mkdir() + (opt / "deepgram").symlink_to(cellar.parent) + detector = layout(opt / "deepgram" / "libexec") + assert detector._detect_method(None, True, False) == InstallMethod.HOMEBREW +@unix_only class TestUvToolDetection: """Test uv tool installation detection.""" - @pytest.fixture - def detector(self): - return InstallationDetector() + def test_custom_uv_tool_dir(self, layout, monkeypatch): + monkeypatch.setenv("UV_TOOL_DIR", "/custom/uv/tools") + assert layout("/custom/uv/tools/deepctl")._is_uv_tool_install() is True - @unix_only - @patch("deepctl_cmd_update.installation.sys") - @patch.dict( - "os.environ", {"UV_TOOL_DIR": "/custom/uv/tools"}, clear=False + def test_default_uv_tool_path(self, layout, monkeypatch): + """Detect uv tool at default location via sys.prefix (symlinked exe).""" + monkeypatch.setattr( + sys, "executable", "/x/.local/share/uv/python/cpython-3.11/bin/python3" + ) + prefix = layout.home / ".local" / "share" / "uv" / "tools" / "deepctl" + assert layout(prefix)._is_uv_tool_install() is True + + def test_xdg_data_home(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + prefix = tmp_path / "data" / "uv" / "tools" / "deepctl" + assert layout(prefix)._is_uv_tool_install() is True + + def test_tools_dir_itself_is_not_a_tool(self, layout): + prefix = layout.home / ".local" / "share" / "uv" / "tools" + assert layout(prefix)._is_uv_tool_install() is False + + +@unix_only +class TestPipxDetection: + def test_pipx_home_env(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + assert layout(tmp_path / "px" / "venvs" / "deepctl")._is_pipx_install() + + @pytest.mark.parametrize( + "parts", [(".local", "pipx"), (".local", "share", "pipx")] ) - def test_custom_uv_tool_dir(self, mock_sys): - """Detect uv tool with custom UV_TOOL_DIR via sys.prefix.""" - mock_sys.executable = "/usr/local/share/uv/python/cpython-3.11/bin/python3" - mock_sys.prefix = "/custom/uv/tools/deepctl" - detector = InstallationDetector() - assert detector._is_uv_tool_install() is True + def test_default_homes(self, layout, parts): + prefix = layout.home.joinpath(*parts, "venvs", "deepctl") + assert layout(prefix)._is_pipx_install() is True - @patch("deepctl_cmd_update.installation.sys") - @patch.dict("os.environ", {}, clear=False) - def test_default_uv_tool_path(self, mock_sys): - """Detect uv tool at default location via sys.prefix (symlinked exe).""" - home = str(Path.home()) - # Simulate the real scenario: exe resolves to the shared Python pool - # (outside tools/), but prefix correctly points into tools/ - mock_sys.executable = f"{home}/.local/share/uv/python/cpython-3.11/bin/python3" - mock_sys.prefix = f"{home}/.local/share/uv/tools/deepctl" - detector = InstallationDetector() - assert detector._is_uv_tool_install() is True + def test_pipx_home_set_for_another_install(self, layout, monkeypatch, tmp_path): + """PIPX_HOME in the environment doesn't make a venv a pipx install.""" + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + monkeypatch.setenv("PIPX_BIN_DIR", str(tmp_path / "bin")) + detector = layout(tmp_path / "project" / ".venv") + assert detector._is_pipx_install() is False +@unix_only class TestOneshotDetection: """Test one-shot execution detection (uvx / pipx run).""" - @pytest.fixture - def detector(self): - return InstallationDetector() + def test_uvx_default_cache(self, layout): + prefix = layout.home / ".cache" / "uv" / "archive-v0" / "AbC123" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.UVX - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3"}, - clear=False, - ) - def test_uvx_detected(self, detector): - """uvx is detected via environment variable.""" - result = detector._detect_oneshot_execution() - assert result == InstallMethod.UVX + def test_uvx_with_uv_cache_dir(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uvc")) + prefix = tmp_path / "uvc" / "archive-v0" / "AbC123" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.UVX + + def test_uv_cache_outside_archive_is_not_uvx(self, layout): + prefix = layout.home / ".cache" / "uv" / "builds-v0" / "x" + assert layout(prefix)._detect_oneshot_execution() is None + + def test_parent_interpreter_env_alone_is_not_uvx(self, layout, monkeypatch, tmp_path): + """uv run sets this too, for an ordinary project venv.""" + monkeypatch.setenv("UV_INTERNAL__PARENT_INTERPRETER", "/usr/bin/python3") + assert layout(tmp_path / "project" / ".venv")._detect_oneshot_execution() is None + + def test_pipx_run_detected(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + prefix = tmp_path / "px" / ".cache" / "0123abcd" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.PIPX_RUN + + def test_no_oneshot(self, layout, tmp_path): + assert layout(tmp_path / "venv")._detect_oneshot_execution() is None - @unix_only - @patch("deepctl_cmd_update.installation.sys") - @patch.dict("os.environ", {}, clear=False) - def test_pipx_run_detected(self, mock_sys, tmp_path): - """pipx run is detected via cache path.""" - # Create a real file so Path.resolve() works - fake_exe = tmp_path / ".local" / "pipx" / ".cache" / "some" / "bin" / "python3" - fake_exe.parent.mkdir(parents=True) - fake_exe.touch() - mock_sys.executable = str(fake_exe) - with patch.dict( - "os.environ", {"PIPX_HOME": str(tmp_path / ".local" / "pipx")}, - clear=False, - ): - detector = InstallationDetector() - result = detector._detect_oneshot_execution() - assert result == InstallMethod.PIPX_RUN - - @patch.dict("os.environ", {}, clear=False) - def test_no_oneshot(self, detector): - """Regular execution returns None.""" - # Make sure UV_INTERNAL__PARENT_INTERPRETER is not set - import os - os.environ.pop("UV_INTERNAL__PARENT_INTERPRETER", None) - result = detector._detect_oneshot_execution() - assert result is None + +@unix_only +class TestNpmLauncher: + @pytest.mark.parametrize("value", ["npm", "npx"]) + def test_env(self, layout, monkeypatch, tmp_path, value): + monkeypatch.setenv("DEEPCTL_LAUNCHER", value) + detector = layout(tmp_path / "cache" / "venv") + assert detector._detect_method(None, True, False) == InstallMethod.NPM + + def test_captured_by_main(self, layout, monkeypatch, tmp_path): + """deepctl.main removes the variable at startup and keeps its value.""" + import deepctl.main as main + + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", True) + monkeypatch.setattr(main, "LAUNCHER", "npm") + detector = layout(tmp_path / "cache" / "venv") + assert detector._detect_method(None, True, False) == InstallMethod.NPM + + def test_unknown_value_ignored(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("DEEPCTL_LAUNCHER", "/usr/local/bin/dg") + detector = layout(tmp_path / "venv") + assert detector._detect_method(None, True, False) != InstallMethod.NPM + + def test_npm_prints_the_command(self, layout, monkeypatch, tmp_path): + from deepctl_cmd_update.installation import InstallationInfo + + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npm") + detector = layout(tmp_path / "venv") + assert detector.get_update_command(InstallMethod.NPM) is None + info = InstallationInfo( + method=InstallMethod.NPM, + path="/x", + virtual_env=True, + editable=False, + python_executable="/x/bin/python3", + ) + assert detector.get_update_instructions(info) == ( + "Run: npm i -g @deepgram/cli@latest" + ) + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npx") + assert "npx" in detector.get_update_instructions(info) class TestGetUpdateInstructions: @@ -268,7 +333,7 @@ def test_uvx_instructions(self, detector): ) instructions = detector.get_update_instructions(info) assert "uvx" in instructions - assert "automatically" in instructions + assert "uv tool install deepctl@latest" in instructions def test_pipx_run_instructions(self, detector): from deepctl_cmd_update.installation import InstallationInfo diff --git a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py index 531e6ffa..4d7b2684 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py @@ -99,16 +99,25 @@ def test_travis(self): class TestOneshotDetection: """Test one-shot execution detection.""" - def test_not_oneshot(self, monkeypatch): - monkeypatch.delenv("UV_INTERNAL__PARENT_INTERPRETER", raising=False) + def test_not_oneshot(self, monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("UV_CACHE_DIR", raising=False) + monkeypatch.delenv("PIPX_HOME", raising=False) + monkeypatch.delenv("DEEPCTL_LAUNCHER", raising=False) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "venv")) assert _is_oneshot() is False - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3"}, - clear=True, - ) - def test_uvx_oneshot(self): + def test_uvx_oneshot(self, monkeypatch, tmp_path): + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uvc")) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "uvc" / "archive-v0" / "x")) + assert _is_oneshot() is True + + def test_npx_oneshot(self, monkeypatch, tmp_path): + main = sys.modules.get("deepctl.main") + if main is not None: + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", False, raising=False) + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npx") + monkeypatch.setattr(sys, "prefix", str(tmp_path / "venv")) assert _is_oneshot() is True diff --git a/packages/deepctl-cmd-update/tests/unit/test_update_command.py b/packages/deepctl-cmd-update/tests/unit/test_update_command.py index 8e9c8246..c6a2f406 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_update_command.py +++ b/packages/deepctl-cmd-update/tests/unit/test_update_command.py @@ -587,3 +587,85 @@ def test_failed_version_check_exits_one( assert command.exit_code_for(result) == 1 assert result.status == "error" assert "connection refused" in result.message + + +class TestDryRun: + """`dg update --dry-run` reports the method and command, and nothing else.""" + + @pytest.mark.parametrize( + ("method", "expected"), + [ + (InstallMethod.PIPX, "Run: pipx upgrade deepctl"), + (InstallMethod.UV_TOOL, "Run: uv tool install deepctl@latest"), + (InstallMethod.HOMEBREW, "Run: brew upgrade deepgram"), + ], + ) + @patch("deepctl_cmd_update.command.VersionChecker") + @patch("deepctl_cmd_update.command.subprocess.run") + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_prints_the_command( + self, detect, run, checker, method, expected, capsys, monkeypatch + ): + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "format", "default") + monkeypatch.setitem(output._output_config, "quiet", False) + detect.return_value = InstallationInfo( + method=method, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + config = MagicMock() + command = UpdateCommand() + result = command.handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert result["message"] == expected + assert result["installation_method"] == method.value + assert command.exit_code_for(result) == 0 + out = capsys.readouterr().out + assert expected in out + assert f"Installation method: {method.value}" in out + checker.assert_not_called() + run.assert_not_called() + config.save.assert_not_called() + config._set_config_value.assert_not_called() + + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_dry_run_leaves_the_config_byte_identical( + self, detect, tmp_path, monkeypatch + ): + config, path = _real_config(tmp_path, monkeypatch) + before = (path.read_bytes(), path.stat().st_mtime_ns) + detect.return_value = InstallationInfo( + method=InstallMethod.PIPX, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + UpdateCommand().handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_json_keeps_stdout_for_the_payload(self, detect, capsys, monkeypatch): + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "format", "json") + detect.return_value = InstallationInfo( + method=InstallMethod.PIPX, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + result = UpdateCommand().handle( + config=MagicMock(), auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert result["message"] == "Run: pipx upgrade deepctl" + assert capsys.readouterr().out == "" diff --git a/src/deepctl/_manifest.d/update.json b/src/deepctl/_manifest.d/update.json index 5e98d438..f292ae7a 100644 --- a/src/deepctl/_manifest.d/update.json +++ b/src/deepctl/_manifest.d/update.json @@ -8,7 +8,7 @@ "requires_auth": false, "help": { "name": "update", - "help": "Check for and install updates to deepctl\n\nExamples:\n dg update\n dg update --check-only\n\n\nFor agents, use --agent-friendly.", + "help": "Check for and install updates to deepctl\n\nExamples:\n dg update\n dg update --check-only\n dg update --dry-run\n\n\nFor agents, use --agent-friendly.", "short_help": "Check for and install updates to deepctl", "epilog": null, "hidden": false, @@ -59,6 +59,27 @@ "hidden": false, "help": "Force update even if already up to date" }, + { + "class": "option", + "name": "dry_run", + "opts": [ + "--dry-run" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Show how deepctl was installed and the command that updates it, without checking for or installing updates" + }, { "class": "option", "name": "check_only", From cbe06c7f3026b791ef463af1a62bf7d26fb60809 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sun, 1 Nov 2026 13:16:36 -0500 Subject: [PATCH 8/8] fix(update): report an unreachable index as an error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When neither PyPI nor, for Homebrew, the GitHub release answers, dg update used to report the running version as the latest. It now exits 1 with update_check_failed (E-UPDATE-CHECK, "Couldn't check for updates: couldn't reach pypi.org.") through the coded error path. On stderr, agent mode shows ERROR: and HINT: lines; human mode shows the error after ✗, followed by the hint. Under -o json, stdout carries the coded payload with details.host. The failed attempt is recorded in the update cache file like any other. --- .../src/deepctl_cmd_update/command.py | 18 +++++++++- .../src/deepctl_cmd_update/version_check.py | 29 +++++++++------ .../tests/unit/test_update_command.py | 35 +++++++++++++++++++ .../tests/unit/test_version_check.py | 30 ++++++++++++++++ .../src/deepctl_core/transport.py | 15 ++++++++ 5 files changed, 116 insertions(+), 11 deletions(-) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py index 4ced9a69..b66df1da 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py @@ -20,7 +20,11 @@ from .installation import InstallationDetector from .models import UpdateResult -from .version_check import VersionChecker, format_version_message +from .version_check import ( + UpdateCheckError, + VersionChecker, + format_version_message, +) # asyncio (and with it ssl) loads only when an update check runs, so @@ -120,9 +124,12 @@ def handle( version_checker = VersionChecker(config, current_version) # Check for updates + unreachable: UpdateCheckError | None = None with console.status("Checking for updates..."): try: version_info = asyncio.run(version_checker.check_version(force=True)) + except UpdateCheckError as e: + unreachable = e except Exception as e: print_error(f"Failed to check for updates: {e}") # status="error" on the model, not a dict: see the docstring. @@ -132,6 +139,15 @@ def handle( success=False, message=f"Failed to check for updates: {e}", ) + if unreachable is not None: + # Never "you're on the latest version": that's unknown. A coded + # error (update_check_failed, exit 1) with ERROR/HINT on stderr. + from deepctl_core.transport import TransportError + + host = unreachable.host + raise TransportError( + "update_check_failed", values={"host": host}, details={"host": host} + ) # Display version info message = format_version_message(version_info) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py index f67e39ed..d46b9f43 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py @@ -16,6 +16,16 @@ from . import check_cache +PYPI_HOST = "pypi.org" + + +class UpdateCheckError(Exception): + """No version source answered; ``host`` is the last one tried.""" + + def __init__(self, host: str) -> None: + self.host = host + super().__init__(f"couldn't reach {host}") + class VersionInfo(BaseModel): """Version information from PyPI.""" @@ -83,6 +93,10 @@ async def check_version( Returns: Version information including update availability + + Raises: + UpdateCheckError: no source answered (PyPI is the last one + tried). Never reported as "up to date". """ # Check if we should skip based on cache if not force and not self.should_check(): @@ -102,16 +116,11 @@ async def check_version( self._cache_info(version_info) return version_info - check_cache.write( - check_cache.UPDATE_FILE, ok=False, latest=None, source=self._source - ) - - # On error, return current version with no update - return VersionInfo( - current_version=self.current_version, - latest_version=self.current_version, - update_available=False, - ) + if check_cache.usable(): + check_cache.write( + check_cache.UPDATE_FILE, ok=False, latest=None, source=self._source + ) + raise UpdateCheckError(PYPI_HOST) async def _check_pypi(self) -> VersionInfo | None: """Fetch latest version from PyPI. diff --git a/packages/deepctl-cmd-update/tests/unit/test_update_command.py b/packages/deepctl-cmd-update/tests/unit/test_update_command.py index c6a2f406..924403a1 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_update_command.py +++ b/packages/deepctl-cmd-update/tests/unit/test_update_command.py @@ -669,3 +669,38 @@ def test_json_keeps_stdout_for_the_payload(self, detect, capsys, monkeypatch): ) assert result["message"] == "Run: pipx upgrade deepctl" assert capsys.readouterr().out == "" + + +class TestUnreachableIndex: + @patch("deepctl_cmd_update.command.asyncio.run") + @patch("deepctl_cmd_update.command.VersionChecker") + def test_raises_update_check_failed(self, checker, run): + from deepctl_cmd_update.version_check import UpdateCheckError + from deepctl_core.transport import TransportError + + run.side_effect = UpdateCheckError("pypi.org") + with pytest.raises(TransportError) as raised: + UpdateCommand().handle( + config=MagicMock(), auth_manager=MagicMock(), client=MagicMock() + ) + err = raised.value + assert err.code == "update_check_failed" + assert err.details == {"host": "pypi.org"} + assert err.error == "Couldn't check for updates: couldn't reach pypi.org." + + def test_renders_with_both_hints(self, monkeypatch): + from deepctl_core import output, transport + + err = transport.TransportError( + "update_check_failed", values={"host": "pypi.org"}, details={"host": "pypi.org"} + ) + monkeypatch.setitem(output._output_config, "agentic", False) + human = transport.render(err, command_path="update") + assert human.code == "update_check_failed" + assert human.hint.startswith("Check your network connection") + assert human.next_step is None + assert human.details == {"host": "pypi.org"} + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setenv("AI_AGENT", "x") + agent = transport.render(err, command_path="update") + assert agent.hint.startswith("The update check needs network access") diff --git a/packages/deepctl-cmd-update/tests/unit/test_version_check.py b/packages/deepctl-cmd-update/tests/unit/test_version_check.py index bde34013..2954d988 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_version_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_version_check.py @@ -383,3 +383,33 @@ def test_legacy_keys_read_when_no_cache_file(self): ) info = VersionChecker(config, "1.0.0")._get_cached_info() assert info is not None and info.latest_version == "1.5.0" + + +class TestUnreachable: + """No answer is an error, never "you are on the latest version".""" + + @staticmethod + def _config(): + config = MagicMock() + config.get.side_effect = lambda key, default=None: ( + True if key == "update.check_enabled" else default + ) + return config + + @pytest.mark.asyncio + @pytest.mark.parametrize("install_method", [None, "homebrew"]) + async def test_raises_and_records_the_failure(self, install_method): + from deepctl_cmd_update import check_cache + from deepctl_cmd_update.version_check import UpdateCheckError + + client = AsyncMock() + client.get.side_effect = httpx.ConnectError("unreachable") + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + checker = VersionChecker(self._config(), "1.0.0") + with patch("httpx.AsyncClient", return_value=client): + with pytest.raises(UpdateCheckError) as raised: + await checker.check_version(force=True, install_method=install_method) + assert raised.value.host == "pypi.org" + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None and state["ok"] is False diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py index 20ae613c..6b4293f8 100644 --- a/packages/deepctl-core/src/deepctl_core/transport.py +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -458,6 +458,15 @@ def sdk_client(self, api_key: str) -> Any: "Retry once after a few seconds. If it fails again, tell the user and " "include details.request_id." ), + # dg update: the package index didn't answer. + "E-UPDATE-CHECK": "Couldn't check for updates: couldn't reach {host}.", + "H-UPDATE-CHECK": ( + "Check your network connection, or update {prog} through the tool you " + "installed it with." + ), + "A-UPDATE-CHECK": ( + "The update check needs network access. Tell the user; don't retry in a loop." + ), } # code -> (error message ID, human hint, agent hint, automation hint or None) @@ -492,6 +501,12 @@ def sdk_client(self, api_key: str) -> Any: None, ), "server_error": ("E-SERVER", "H-SERVER", "A-SERVER", None), + "update_check_failed": ( + "E-UPDATE-CHECK", + "H-UPDATE-CHECK", + "A-UPDATE-CHECK", + None, + ), } # network_unavailable's other messages: the connection dropped after the