diff --git a/.github/workflows/publish-on-release.yml b/.github/workflows/publish-on-release.yml deleted file mode 100644 index 673795d3..00000000 --- a/.github/workflows/publish-on-release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Publish to PyPI on release - -# Fires when a human publishes a GitHub release with `gh release create -# v0.X.Y` (or via the GitHub UI) to recover from a stuck release-please -# state. Typical trigger: release-please aborts with `untagged, merged -# release PRs outstanding` because a previous merged release PR didn't -# get tagged, and `autorelease: pending` is stuck on it. -# -# Releases created by GITHUB_TOKEN (release-please's normal flow) do NOT -# trigger this workflow because GitHub Actions intentionally blocks that -# loop, so the standard release-please path keeps using release.yml's -# existing publish job. This workflow ONLY fires on human-driven -# recovery releases, so there is no double-publish race. -# -# Intentionally NO workflow_dispatch trigger. Manual deploys outside a -# real release-published event have shipped landing pages and packages -# that pointed at unreleased versions. Production publish must always -# be anchored to a real GitHub release artifact. - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - publish: - name: Build and publish to PyPI - runs-on: ubuntu-latest - # Only fire on root-package releases (v0.2.4, v1.0.0, …). - # Sub-package tags look like deepctl-cmd-listen-v0.0.3, which we skip; - # release-please's normal flow handles those via release.yml. - if: startsWith(github.event.release.tag_name, 'v') - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - with: - ref: ${{ github.event.release.tag_name }} - - - name: Set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: "3.11" - - - name: Install build dependencies - run: pip install build twine - - - name: Build all packages - run: make build - - - name: Verify built packages - run: make verify-packages - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: dist/ - skip-existing: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4400384..af67ace9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,12 +4,36 @@ on: push: branches: - main + # A dispatch with no tag runs release-please, like a push. A dispatch + # with a tag rebuilds and publishes that existing release instead: the + # way to recover when a release was tagged but never reached PyPI. workflow_dispatch: + inputs: + tag: + description: >- + Existing root release tag (vX.Y.Z) to rebuild and publish. + Leave empty to run release-please. + type: string + required: false + default: "" + index: + description: >- + Where a tag dispatch publishes. testpypi is the dry run of the + trusted-publishing setup and advertises nothing. + type: choice + options: + - pypi + - testpypi + default: pypi permissions: contents: write pull-requests: write +# Keep CI from sending telemetry from anything this workflow runs. +env: + DEEPCTL_TELEMETRY_DISABLED: "1" + jobs: release-please: name: Release Please @@ -26,6 +50,16 @@ jobs: token: ${{ github.token }} config-file: .github/release-please-config.json manifest-file: .github/.release-please-manifest.json + # Only a push or a plain dispatch cuts a release. A tag dispatch + # republishes an existing tag, and a TestPyPI dry run must not + # create a real release. + if: ${{ github.event_name == 'push' || (inputs.tag == '' && inputs.index != 'testpypi') }} + + - name: Refuse a TestPyPI dry run without a tag + if: ${{ github.event_name == 'workflow_dispatch' && inputs.index == 'testpypi' && inputs.tag == '' }} + run: | + echo "::error::a TestPyPI dry run needs a tag to publish" + exit 1 # The release PR's re-pin runs as two jobs so that no third-party code ever # runs next to a write credential: @@ -218,10 +252,54 @@ jobs: build: name: Build packages needs: release-please - if: ${{ needs.release-please.outputs.release_created }} + if: >- + ${{ needs.release-please.outputs.release_created == 'true' || + (github.event_name == 'workflow_dispatch' && inputs.tag != '') }} runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + persist-credentials: false + # A new release builds the tag release-please just created, not + # whatever the run's branch points at. A recovery checks out its + # tag below, and needs the branch history and its tags for that. + ref: ${{ (github.event_name != 'workflow_dispatch' || inputs.tag == '') && needs.release-please.outputs.tag_name || '' }} + fetch-depth: ${{ (github.event_name == 'workflow_dispatch' && inputs.tag != '') && '0' || '1' }} + + # Only a root release tag that is already a published GitHub release + # and is in the history of the branch this run was started on. The + # publish environments accept only main, release/* and next/*. + - name: Check out the recovery tag + if: ${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }} + env: + TAG: ${{ inputs.tag }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Checked before TAG is printed anywhere. + if ! [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::the tag input is not a root release tag (vX.Y.Z)"; exit 1 + fi + case "${GITHUB_REF}" in + refs/heads/main | refs/heads/release/* | refs/heads/next/*) ;; + *) echo "::error::start a recovery from main, release/* or next/*, not ${GITHUB_REF}"; exit 1 ;; + esac + if ! git rev-parse -q --verify "refs/tags/${TAG}^{commit}" >/dev/null; then + echo "::error::no tag ${TAG}"; exit 1 + fi + if ! git merge-base --is-ancestor "refs/tags/${TAG}^{commit}" "${GITHUB_SHA}"; then + echo "::error::${TAG} is not in the history of ${GITHUB_REF_NAME}"; exit 1 + fi + if ! DRAFT=$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft); then + echo "::error::${TAG} has no GitHub release"; exit 1 + fi + if [ "${DRAFT}" != "false" ]; then + echo "::error::${TAG}'s GitHub release is still a draft"; exit 1 + fi + git checkout --detach "refs/tags/${TAG}^{commit}" + echo "Rebuilding ${TAG} at $(git rev-parse HEAD)" - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -237,6 +315,20 @@ jobs: - name: Verify built packages run: make verify-packages + # A tag name can move; the version inside the build can't. Publish + # only if the tree that was built carries the tag's version. + - name: Check the build matches the release tag + if: ${{ startsWith(inputs.tag || needs.release-please.outputs.tag_name, 'v') }} + env: + TAG: ${{ inputs.tag || needs.release-please.outputs.tag_name }} + run: | + set -euo pipefail + if ! compgen -G "dist/deepctl-${TAG#v}-*.whl" >/dev/null; then + echo "::error::the build has no deepctl ${TAG#v} wheel; ${TAG} does not point at that release" + ls dist/ + exit 1 + fi + - name: Upload artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -247,8 +339,12 @@ jobs: name: Test installation needs: build runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -373,10 +469,62 @@ jobs: uses: ./.github/workflows/web-production.yml secrets: inherit + # Trusted Publishing: each of the 31 PyPI projects names this file and + # the pypi environment as its publisher, and PyPI swaps the job's OIDC + # token for a short-lived upload token. There is no API token to read. + # The pypi environment deploys only from main, release/* and next/*. publish: name: Publish to PyPI needs: [build, test] + if: ${{ github.event_name == 'push' || inputs.index == 'pypi' }} + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/deepctl/ + permissions: + id-token: write + steps: + - name: Download artifacts + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + + # The root package goes last: it pins every sub-package ==, so it must + # never be on the index before them. "deepctl-" sorts before + # "deepctl_", so one upload of dist/ would send it first. + - name: Hold the root package back + run: | + set -euo pipefail + mkdir root-dist + mv dist/deepctl-[0-9]* root-dist/ + ls root-dist/ + + - name: Publish the sub-packages to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/ + skip-existing: true + + - name: Publish the root package to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: root-dist/ + skip-existing: true + + # The same upload against TestPyPI, from its own environment so that its + # OIDC token can never match a production PyPI publisher. Nothing + # downstream needs this job, so a dry run advertises nothing. + publish-testpypi: + name: Publish to TestPyPI + needs: [build, test] + if: ${{ github.event_name == 'workflow_dispatch' && inputs.index == 'testpypi' }} runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/deepctl/ + permissions: + id-token: write steps: - name: Download artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 @@ -384,13 +532,28 @@ jobs: name: dist path: dist/ - - name: Publish to PyPI + # Root package last, as in publish. + - name: Hold the root package back + run: | + set -euo pipefail + mkdir root-dist + mv dist/deepctl-[0-9]* root-dist/ + ls root-dist/ + + - name: Publish the sub-packages to TestPyPI uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: - password: ${{ secrets.PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ packages-dir: dist/ skip-existing: true + - name: Publish the root package to TestPyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: root-dist/ + skip-existing: true + bump-brew-formula: name: Bump Homebrew formula needs: [release-please, publish, verify-published] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed2ea345..c81b71d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,11 @@ jobs: run: | uv run pytest tests packages/deepctl-core/tests packages/deepctl-shared-utils/tests packages/deepctl-telemetry/tests packages/deepctl-cmd-api/tests packages/deepctl-cmd-login/tests packages/deepctl-cmd-projects/tests packages/deepctl-cmd-transcribe/tests packages/deepctl-cmd-usage/tests packages/deepctl-cmd-debug/tests packages/deepctl-cmd-debug-audio/tests packages/deepctl-cmd-debug-browser/tests packages/deepctl-cmd-debug-network/tests packages/deepctl-cmd-debug-toolkit/tests packages/deepctl-cmd-debug-probe/tests packages/deepctl-cmd-ffprobe/tests packages/deepctl-cmd-mcp/tests packages/deepctl-cmd-update/tests packages/deepctl-cmd-plugin/tests packages/deepctl-cmd-skills/tests packages/deepctl-cmd-init/tests packages/deepctl-cmd-models/tests packages/deepctl-cmd-speak/tests packages/deepctl-cmd-keys/tests packages/deepctl-cmd-read/tests packages/deepctl-cmd-listen/tests packages/deepctl-cmd-requests/tests packages/deepctl-cmd-billing/tests packages/deepctl-cmd-members/tests packages/deepctl-cmd-completion/tests packages/deepctl-plugin-example/tests --cov --cov-report=term-missing -v + # Outside pytest, so conftest.py's telemetry opt-out doesn't apply. - name: Test CLI installation + env: + DEEPCTL_TELEMETRY_DISABLED: "1" + DEEPCTL_TELEMETRY_DSN: "http://k@127.0.0.1:9/1" run: | uv run python -c "from deepctl.main import main; print('Import successful')" uv run deepctl --version diff --git a/conftest.py b/conftest.py index 86d564bc..5ae8166f 100644 --- a/conftest.py +++ b/conftest.py @@ -88,3 +88,14 @@ def _restore_sigint(): yield if before is not None: signal.signal(signal.SIGINT, before) + + +@pytest.fixture(autouse=True) +def _fresh_credentials_state(): + """Forget per-process resolver state (placeholder warnings, a keyring + timeout) between tests, as each dg run starts without it.""" + from deepctl_core import credentials + + credentials._reset_for_tests() + yield + credentials._reset_for_tests() diff --git a/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py b/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py index 4792f874..43a434f9 100644 --- a/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py +++ b/packages/deepctl-cmd-api/src/deepctl_cmd_api/command.py @@ -1,6 +1,7 @@ """API command for deepctl.""" import json +import os import shutil import subprocess import sys @@ -288,14 +289,22 @@ def _build_headers( custom_headers: tuple[str, ...] | list[str] | None, ) -> dict[str, str]: """Build request headers with auth (when given) and custom headers.""" + from deepctl_core.transport import request_headers + headers: dict[str, str] = {} if api_key: - headers["Authorization"] = f"Token {api_key}" + # The key's host passed the check; the session ID goes only + # where the key goes. + headers = request_headers(api_key) if custom_headers: for header in custom_headers: if ":" in header: key, value = header.split(":", 1) - headers[key.strip()] = value.strip() + key = key.strip() + # A -H header replaces one of the same name, in any case. + for name in [k for k in headers if k.lower() == key.lower()]: + del headers[name] + headers[key] = value.strip() return headers def _build_body( @@ -363,6 +372,25 @@ def _apply_jq_filter(self, response_text: str, jq_filter: str) -> str: raise RuntimeError(f"jq error: {result.stderr.strip()}") return result.stdout.strip() + @staticmethod + def _insecure_ignored(url: str, error: BaseException) -> bool: + """A TLS failure on a loopback host while DEEPGRAM_CLI_INSECURE would + have skipped verification for the SDK: say it doesn't reach dg api.""" + from deepctl_core import auth, output, transport + + if output.is_ai_agent(): + return False + value = os.environ.get("DEEPGRAM_CLI_INSECURE", "").strip().lower() + origin = parse_origin(url) + if value not in ("1", "true") or origin is None: + return False + if origin.host not in auth._INSECURE_HOSTS: + return False + mapped = transport.map_exception(error) + return ( + mapped is not None and mapped.details.get("reason") == transport.REASON_TLS + ) + def handle( self, config: Config, @@ -382,9 +410,14 @@ def handle( raw_output = kwargs.get("raw", False) allow_host = kwargs.get("allow_foreign_host") - # Resolve URL against the profile's base URL. check_base_url raises - # (already reported) when that host isn't trusted with the key. - base_url = auth_manager.check_base_url() + from deepctl_core import transport + + # Resolve URL against the profile's base URL. The Transport's host + # check raises (already reported) when that host isn't trusted with + # the key. + base_url = transport.Transport( + config, check_host=auth_manager.check_base_url + ).rest_base() url = self._resolve_url(endpoint, base_url) attach, refusal = self._authorize(url, base_url, config, allow_host, method) @@ -432,11 +465,17 @@ def handle( # Make request try: - # Never follow redirects: a 3xx could point the key elsewhere. - with httpx.Client(timeout=30.0, follow_redirects=False) as http_client: - response = http_client.request(**request_kwargs) + # The process's one client (one connection), with TLS verified + # for every host as in 0.4.1: DEEPGRAM_CLI_INSECURE doesn't reach + # dg api. Never follow redirects: a 3xx could point the key + # elsewhere. + response = transport.verified_client().request( + **request_kwargs, follow_redirects=False, timeout=30.0 + ) except httpx.RequestError as e: console.print(f"[red]Request failed: {e}[/red]") + if self._insecure_ignored(url, e): + print_hint(msg.H_API_INSECURE_IGNORED) return ApiResult( status="error", message=f"Request failed: {e}", diff --git a/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py b/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py index 24fabd51..79f3e184 100644 --- a/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py +++ b/packages/deepctl-cmd-api/src/deepctl_cmd_api/messages.py @@ -29,3 +29,9 @@ "{origin} answered with a redirect to {location_origin}. dg api doesn't " "follow redirects." ) +# After a TLS failure on localhost, 127.0.0.1 or [::1] while a person has +# DEEPGRAM_CLI_INSECURE set: dg api always verifies TLS (as in 0.4.1). +H_API_INSECURE_IGNORED = ( + "DEEPGRAM_CLI_INSECURE doesn't apply to dg api, which always verifies TLS " + "certificates." +) diff --git a/packages/deepctl-cmd-api/tests/unit/test_api_command.py b/packages/deepctl-cmd-api/tests/unit/test_api_command.py index 046c3447..c6542cdf 100644 --- a/packages/deepctl-cmd-api/tests/unit/test_api_command.py +++ b/packages/deepctl-cmd-api/tests/unit/test_api_command.py @@ -1,6 +1,8 @@ """Tests for API command.""" import json +import os +from pathlib import Path from unittest.mock import Mock, patch import httpx @@ -93,9 +95,22 @@ def test_build_headers(self, command): assert headers["Accept"] == "text/plain" def test_build_headers_no_custom(self, command): - """Test header building with only auth.""" + """With a key: the transport's request headers (identification, + session ID, auth), as every other API request sends.""" + from deepctl_core import transport + headers = command._build_headers("test-key", None) - assert headers == {"Authorization": "Token test-key"} + assert headers == transport.request_headers("test-key") + assert headers["Authorization"] == "Token test-key" + + def test_build_headers_without_a_key_has_no_session_id(self, command): + """No key (a host that doesn't get it): nothing from the builder.""" + assert command._build_headers(None, ["X-A: 1"]) == {"X-A": "1"} + + def test_custom_header_replaces_one_in_any_case(self, command): + headers = command._build_headers("test-key", ["user-agent: mine"]) + assert [k for k in headers if k.lower() == "user-agent"] == ["user-agent"] + assert headers["user-agent"] == "mine" def test_build_body_from_fields(self, command): """Test body building from field arguments.""" @@ -175,7 +190,7 @@ def test_apply_jq_filter_error(self, mock_run, mock_which, command): with pytest.raises(RuntimeError, match="jq error"): command._apply_jq_filter("{}", "invalid[") - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_success( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -187,8 +202,6 @@ def test_handle_success( mock_response.elapsed.total_seconds.return_value = 0.150 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -206,7 +219,7 @@ def test_handle_success( assert result.url == f"{DEFAULT_BASE_URL}/v1/projects" assert result.response_body == {"projects": []} - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_error_response( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -218,8 +231,6 @@ def test_handle_error_response( mock_response.elapsed.total_seconds.return_value = 0.050 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -234,14 +245,12 @@ def test_handle_error_response( assert result.status == "error" assert result.status_code == 404 - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_request_error( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): """Test handling of request errors.""" mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.side_effect = httpx.ConnectError("Connection refused") mock_client_class.return_value = mock_http @@ -275,7 +284,7 @@ def test_handle_no_api_key( assert result.status == "error" assert "No API key found" in result.message - @patch("httpx.Client") + @patch("deepctl_core.transport.shared_client") def test_handle_post_with_fields( self, mock_client_class, command, mock_config, mock_auth_manager, mock_client ): @@ -287,8 +296,6 @@ def test_handle_post_with_fields( mock_response.elapsed.total_seconds.return_value = 0.200 mock_http = Mock() - mock_http.__enter__ = Mock(return_value=mock_http) - mock_http.__exit__ = Mock(return_value=False) mock_http.request.return_value = mock_response mock_client_class.return_value = mock_http @@ -357,3 +364,52 @@ def test_api_result_serialization(self): data = result.model_dump() assert data["method"] == "POST" assert data["status_code"] == 201 + + +@pytest.mark.parametrize( + ("insecure", "host", "agent", "hinted"), + [ + ("1", "127.0.0.1", False, True), + ("true", "localhost", False, True), + ("1", "127.0.0.1", True, False), + ("0", "127.0.0.1", False, False), + ("1", "api.eu.deepgram.com", False, False), + ], +) +def test_insecure_hint_on_loopback_tls_failure(monkeypatch, capsys, insecure, host, agent, hinted): + """dg api always verifies TLS; say so when DEEPGRAM_CLI_INSECURE was set.""" + from deepctl_core import output + + base = f"https://{host}:8443" if host != "api.eu.deepgram.com" else f"https://{host}" + auth = Mock(spec=AuthManager) + auth.get_api_key.return_value = "test-api-key" + auth.check_base_url.return_value = base + config = Mock(spec=Config) + config.allowed_hosts.return_value = [f"{host}:8443"] + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", insecure) + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + monkeypatch.setattr(output.stderr_console, "_width", 400) + failing = Mock() + failing.request.side_effect = httpx.ConnectError( + "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed" + ) + monkeypatch.setattr("deepctl_core.transport.verified_client", lambda: failing) + result = ApiCommand().handle(config, auth, Mock(), endpoint="/v1/projects") + assert result.status == "error" + err = capsys.readouterr().err + assert ("DEEPGRAM_CLI_INSECURE doesn't apply to dg api," in err) is hinted + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +def test_insecure_hint_matches_catalogue(): + + from deepctl_cmd_api import messages + + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + assert ( + catalogue["strings"]["H-API-INSECURE-IGNORED"] + == messages.H_API_INSECURE_IGNORED + ) diff --git a/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py b/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py index 0373dacf..4e247c8d 100644 --- a/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py +++ b/packages/deepctl-cmd-api/tests/unit/test_api_host_scope.py @@ -62,15 +62,17 @@ def handler(request: httpx.Request) -> httpx.Response: stream=httpx.ByteStream(b'{"ok": true}'), ) - real_client = httpx.Client + class Shared(httpx.Client): + """The shared client; `dg api` must ask for no redirects per request.""" - def client(**kwargs): - if "follow_redirects" not in kwargs: # not the `dg api` client - return real_client(**kwargs) - assert kwargs["follow_redirects"] is False - return real_client(transport=httpx.MockTransport(handler), **kwargs) + def request(self, *args, **kwargs): + assert kwargs["follow_redirects"] is False + return super().request(*args, **kwargs) - monkeypatch.setattr(api_module.httpx, "Client", client) + from deepctl_core import transport + + monkeypatch.setattr(transport, "_client", None) + transport._set_client(Shared(transport=httpx.MockTransport(handler))) return seen diff --git a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py index d80b0fb8..fab19427 100644 --- a/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py +++ b/packages/deepctl-cmd-debug-network/src/deepctl_cmd_debug_network/command.py @@ -32,11 +32,15 @@ console = Console() -# Default Deepgram endpoints to test -DEEPGRAM_ENDPOINTS = { - "api": "https://api.deepgram.com", - "auth": "https://auth.dx.deepgram.com", -} + +def _default_host() -> str: + """The Deepgram API host (``hosts.DEFAULT_BASE_URL``, via the Transport). + + The default endpoint only: this command sends no credentials anywhere. + """ + from deepctl_core import transport + + return transport.default_host() def _is_ssl_error(exc: BaseException) -> bool: @@ -131,7 +135,7 @@ def get_arguments(self) -> list[dict[str, Any]]: return [ { "names": ["--endpoint", "-e"], - "help": ("Specific endpoint to test (default: api.deepgram.com)"), + "help": f"Specific endpoint to test (default: {_default_host()})", "type": str, "is_option": True, }, @@ -205,7 +209,7 @@ def handle( **kwargs: Any, ) -> Any: """Handle network debug command execution.""" - endpoint = kwargs.get("endpoint") or "api.deepgram.com" + endpoint = kwargs.get("endpoint") or _default_host() verbose = kwargs.get("verbose", False) skip_commands = kwargs.get("skip_commands", False) timeout = kwargs.get("timeout", 10) @@ -744,7 +748,7 @@ def _test_multi_region_latency( ) -> None: """Test latency to known Deepgram endpoints.""" endpoints = [ - ("api.deepgram.com", "Primary API"), + (_default_host(), "Primary API"), ] console.print("\n[bold]Multi-Region Latency:[/bold]") @@ -1054,7 +1058,7 @@ def _generate_recommendations(self, result: NetworkDebugResult) -> None: recommendations.append( "ℹ️ Proxy configuration detected. Ensure your proxy allows access to:" ) - recommendations.append(" • api.deepgram.com (port 443)") + recommendations.append(f" • {_default_host()} (port 443)") recommendations.append(" • Certificate validation endpoints (port 80)") # Display recommendations diff --git a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py index 667a17dd..e795bd60 100644 --- a/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py +++ b/packages/deepctl-cmd-debug-network/tests/unit/test_network_command.py @@ -707,3 +707,17 @@ def test_missing_bundle_is_reported_like_requests( assert with_verify.success is False assert "Could not find a suitable TLS CA certificate bundle" in with_verify.error assert without_verify.success is True + + +def test_default_endpoint_comes_from_hosts_default(): + """The default host is hosts.DEFAULT_BASE_URL's, through the + Transport; debug network sends no credentials anywhere.""" + from pathlib import Path + + from deepctl_cmd_debug_network import command as network_module + from deepctl_core.hosts import DEFAULT_BASE_URL + + assert f"https://{network_module._default_host()}" == DEFAULT_BASE_URL + source = Path(network_module.__file__).read_text(encoding="utf-8") + for credential_use in ("get_api_key", "Authorization", "auth_headers"): + assert credential_use not in source diff --git a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py index fd67c637..8a4c08eb 100644 --- a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py +++ b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/command.py @@ -6,14 +6,20 @@ import time from typing import Any -from deepctl_core import AuthManager, BaseCommand, Config, DeepgramClient +from deepctl_core import ( + AuthManager, + BaseCommand, + Config, + DeepgramClient, + get_status_console, +) from deepctl_shared_utils import require_ffprobe -from rich.console import Console from rich.panel import Panel from .models import ProbeDebugResult -console = Console() +# Status chrome is diagnostics: stderr, so stdout carries only the result. +console = get_status_console() def _find_available_port(start: int = 3100, end: int = 3199) -> int | None: @@ -165,24 +171,33 @@ def handle( port=port, ) - upstream_host = "api.deepgram.com" + from deepctl_core import transport + from rich.markup import escape + + # The upstream is the Transport's checked WebSocket base, and the key + # goes only where that check allows (refused hosts raise, reported). + upstream = transport.Transport(config, check_host=auth_manager.check_base_url) + upstream_base = upstream.ws_base() + upstream_headers = upstream.auth_headers(api_key) # Create proxy proxy = ProbeProxy( api_key=api_key, config=config, - upstream_host=upstream_host, + upstream_base=upstream_base, probe_interval_bytes=probe_interval_bytes, probe_interval_seconds=probe_interval_seconds, verbose=verbose, + upstream_headers=upstream_headers, ) + upstream_host = proxy.upstream_host # Print instructions console.print( Panel( f"[bold cyan]WebSocket Probe Proxy[/bold cyan]\n\n" f"Listening on: [green]ws://localhost:{port}[/green]\n" - f"Upstream: [yellow]wss://{upstream_host}[/yellow]\n" + f"Forwards to: [yellow]{escape(proxy.upstream_base)}[/yellow]\n" f"API key: [dim]****{api_key[-4:]}[/dim]\n\n" f"Probe every: {probe_interval_bytes:,} bytes " f"or {probe_interval_seconds}s\n\n" @@ -205,7 +220,10 @@ def interrupt() -> None: if proxy._stop.is_set(): proxy.abort() else: - console.print("\n[yellow]Shutting down probe proxy...[/yellow]") + console.print( + "\n[yellow]Stopping: open connections can finish. Press " + "Ctrl+C again to end them now.[/yellow]" + ) proxy._stop.set() installed, previous = _handle_sigint(loop, interrupt) @@ -231,6 +249,10 @@ def interrupt() -> None: signal.signal(signal.SIGINT, previous) loop.close() + if proxy.fatal is not None: + # The key was rejected at the upgrade: exit 1 with its code. + raise proxy.fatal + duration = time.time() - start_time console.print(f"\n[green]Probe proxy stopped after {duration:.1f}s[/green]") diff --git a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py index ba2834cd..3a90cad4 100644 --- a/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py +++ b/packages/deepctl-cmd-debug-probe/src/deepctl_cmd_debug_probe/proxy.py @@ -14,12 +14,12 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit +from deepctl_core.output import get_status_console from deepctl_core.websocket import RedirectRefused from deepctl_shared_utils.ffprobe import ( _parse_probe_data, # lockstep sibling; same parsing as probe_buffer get_ffprobe_path, ) -from rich.console import Console from rich.text import Text from websockets.asyncio.server import Server, ServerConnection, serve from websockets.exceptions import ConnectionClosed, WebSocketException @@ -29,11 +29,18 @@ if TYPE_CHECKING: from deepctl_core import Config + from deepctl_core.transport import TransportError from deepctl_shared_utils.ffprobe_models import AudioProbeResult from websockets.asyncio.client import ClientConnection from websockets.http11 import Request, Response -console = Console() +# Status and transcripts are diagnostics: stderr, so stdout carries only the +# command's result (one JSON document under -o json). +console = get_status_console() + +# Refusals that end the proxy: the key is wrong for every connection. A 403 +# is about this request's scope (another path may be allowed), so it isn't. +_FATAL_CODES = frozenset({"auth_invalid"}) # Frames up to 4 MiB each way, the limit earlier releases allowed. MAX_FRAME_BYTES = 4 * 2**20 @@ -71,21 +78,33 @@ def __init__( self, api_key: str, config: Config | None = None, - upstream_host: str = "api.deepgram.com", + upstream_base: str | None = None, probe_interval_bytes: int = 131072, probe_interval_seconds: float = 10.0, verbose: bool = False, + upstream_headers: dict[str, str] | None = None, ) -> None: + """``upstream_base`` and ``upstream_headers`` come from the command's + Transport: the checked WebSocket base and the ``Authorization`` it + attaches only after the host check passed. Without headers, the + upstream gets no key.""" + from deepctl_core import transport + from deepctl_core.hosts import DEFAULT_BASE_URL + self.api_key = api_key self.config = config - self.upstream_host = upstream_host + # Where connections go: the base, then the client's path and query. + self.upstream_base = transport.ws_url(upstream_base or DEFAULT_BASE_URL) + self.upstream_base = self.upstream_base.rstrip("/") + self.upstream_host = urlsplit(self.upstream_base).hostname or "" + self.upstream_headers = dict(upstream_headers or {}) self.probe_interval_bytes = probe_interval_bytes self.probe_interval_seconds = probe_interval_seconds self.verbose = verbose self.connections: list[ProbeConnectionStats] = [] - # Where connections go: wss:// plus the fixed host, then the client's - # path and query. Tests point it at a loopback ws:// server. - self._upstream_base = f"wss://{upstream_host}" + # A coded error from an upstream that refused the key at the upgrade: + # the proxy stops and the command reports it (auth_invalid). + self.fatal: TransportError | None = None self._stop = asyncio.Event() # Open client and upstream connections, and pending upstream # connects, so a second Ctrl+C can end them at once. @@ -139,9 +158,10 @@ def process_request( async def _connect_upstream( self, url: str, headers: dict[str, str] ) -> ClientConnection: - """Open the upstream WebSocket. The API key goes only here, and a - redirect on the upgrade is refused (``RedirectRefused``), never - followed.""" + """Open the upstream WebSocket, within the Transport's open timeout. + The API key goes only here, and a redirect on the upgrade is refused + (``RedirectRefused``, a coded network_unavailable), never followed.""" + from deepctl_core import transport from deepctl_core.websocket import open_async conn: ClientConnection = await open_async( @@ -149,7 +169,7 @@ async def _connect_upstream( headers, command="dg debug probe", payload="audio", - open_timeout=10, + open_timeout=transport.WS_OPEN_TIMEOUT_S, close_timeout=30, max_size=MAX_FRAME_BYTES, compression=None, @@ -188,15 +208,34 @@ def abort(self) -> None: for proc in list(self._ffprobes): self._reaping.add(asyncio.ensure_future(self._stop_ffprobe(proc))) + async def close_all(self) -> None: + """Close every open connection on both sides (each within 2 s).""" + for conn in list(self._live): + with contextlib.suppress(Exception): + await asyncio.wait_for(conn.close(), 2) + + async def _stop_all(self) -> None: + """A fatal stop: close every open stream, then stop serving.""" + await self.close_all() + self._stop.set() + async def run(self, port: int) -> None: """Serve until `_stop` is set or this task is cancelled; then close - the listener and wait for every connection's summary.""" + the listener and wait for every connection's summary. After a fatal + refusal that wait is capped at 2 s; what's left is aborted.""" server = await self.start(port) try: await self._stop.wait() finally: server.close() - await server.wait_closed() + if self.fatal is not None: + closed = asyncio.ensure_future(server.wait_closed()) + done, _ = await asyncio.wait({closed}, timeout=2) + if not done: + self.abort() + await closed + else: + await server.wait_closed() await asyncio.gather(*self._reaping) async def handle_connection(self, ws_client: ServerConnection) -> None: @@ -213,11 +252,11 @@ async def handle_connection(self, ws_client: ServerConnection) -> None: ) self.connections.append(stats) - upstream_url = self._upstream_base + target + upstream_url = self.upstream_base + target - upstream_headers = { - "Authorization": f"Token {self.api_key}", - } + # The Transport's Authorization, attached after its host check; the + # proxy never builds it. + upstream_headers = dict(self.upstream_headers) console.print( Text.from_markup( @@ -241,21 +280,32 @@ async def handle_connection(self, ws_client: ServerConnection) -> None: self._connecting.add(connecting) try: ws_upstream = await connecting - except RedirectRefused as e: - console.print( - Text.from_markup( - f"[red]Upstream connection refused[/red] [{conn_id}]: " - ), - untrusted(e.message), - sep="", + except ( + RedirectRefused, + WebSocketException, + OSError, + asyncio.TimeoutError, + ) as e: + # A refused redirect is already coded (network_unavailable) + # and fails only this connection. + from deepctl_core import transport + + mapped = transport.map_ws_open( + e, (self.api_key,), host=self.upstream_host, path=path ) - return - except (WebSocketException, OSError, asyncio.TimeoutError) as e: + if mapped is not None and mapped.code in _FATAL_CODES: + # The key itself was refused: every connection would fail + # the same way. Stop, and let the command report the coded + # error. Only the first refusal stops. + if self.fatal is None: + self.fatal = mapped + await self._stop_all() + return console.print( Text.from_markup( - f"[red]Upstream connection failed[/red] [{conn_id}]: " + f"[red]Deepgram connection failed[/red] [{conn_id}]: " ), - untrusted(str(e)), + untrusted(f"{mapped.error} ({mapped.code})" if mapped else str(e)), sep="", ) return diff --git a/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py b/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py index 34fa059d..4800b470 100644 --- a/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py +++ b/packages/deepctl-cmd-debug-probe/tests/unit/test_probe.py @@ -1,5 +1,6 @@ """Tests for debug probe command and proxy.""" +import asyncio import json from unittest.mock import Mock, patch @@ -236,8 +237,13 @@ async def target(reader, writer): stubs.append(srv) ports[name] = srv.sockets[0].getsockname()[1] - proxy = ProbeProxy(api_key="fa1ce0000000000000000000000000000000000d") - proxy._upstream_base = f"ws://127.0.0.1:{ports['origin']}" + proxy = ProbeProxy( + api_key="fa1ce0000000000000000000000000000000000d", + upstream_base=f"ws://127.0.0.1:{ports['origin']}", + upstream_headers={ + "Authorization": "Token fa1ce0000000000000000000000000000000000d" + }, + ) server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] try: @@ -256,7 +262,7 @@ async def target(reader, writer): out = io.StringIO() with patch( "deepctl_cmd_debug_probe.proxy.console", - Console(file=out, force_terminal=False, width=300), + Console(file=out, force_terminal=False, width=300, stderr=True), ): asyncio.run(run()) return origin_reqs, target_reqs, out.getvalue() @@ -273,7 +279,8 @@ def test_redirect_refused_and_nothing_reaches_the_target(self, location): assert target == [] assert len(origin) == 1 assert b"Token fa1ce0" in origin[0] - assert "Upstream connection refused" in text + assert "Deepgram connection failed" in text + assert "(network_unavailable)" in text assert "dg debug probe doesn't follow redirects, so no audio was sent." in ( " ".join(text.split()) ) @@ -320,6 +327,8 @@ async def run(self, port): auth = Mock() auth.get_api_key.return_value = "fa1ce0000000000000000000000000000000000d" + # The Transport's host check: the default host passes. + auth.check_base_url.return_value = "https://api.deepgram.com" with ( patch("deepctl_cmd_debug_probe.command.require_ffprobe", return_value=True), patch.object(proxymod.ProbeProxy, "run", run), @@ -353,7 +362,7 @@ def custom(signum, frame): # pragma: no cover - never called signal.signal(signal.SIGINT, old) assert result.status == "success" assert seen == [] - assert "Shutting down probe proxy" in capsys.readouterr().out + assert "Stopping: open connections can finish" in capsys.readouterr().err def test_ignored_sigint_stays_ignored(self): """A background job in a script starts with SIGINT ignored: kill -INT @@ -387,7 +396,7 @@ def hooks(proxy, loop, port): assert out["ignored"] is True if out["sigign"] is not None: # Linux: the SIGINT bit is still set assert int(out["sigign"], 16) & (1 << (2 - 1)) - assert "Shutting down probe proxy" not in proc.stdout + assert "Stopping: open connections can finish" not in proc.stderr def test_keyboard_interrupt_path_first_and_second_press(self, capsys): """Where the loop can't take SIGINT (Windows), Ctrl+C raises @@ -423,7 +432,7 @@ def abort(): assert time.monotonic() - t0 < 10 assert result.status == "success" assert calls == ["abort"] - assert "Shutting down probe proxy" in capsys.readouterr().out + assert "Stopping: open connections can finish" in capsys.readouterr().err async def _hold_connection(proxy, port): @@ -438,7 +447,7 @@ async def never(reader, writer): writer.close() hang = await asyncio.start_server(never, "127.0.0.1", 0) - proxy._upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" + proxy.upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" for _ in range(100): try: ws = await connect(f"ws://127.0.0.1:{port}/v1/listen") @@ -520,3 +529,213 @@ def test_localhost_host_allowed_any_case(self): ) def test_foreign_host_refused(self, host): assert self._status([("Host", host)]) == 403 + + +# ── The upstream comes from the Transport ── + +KEY = "fa1ce0000000000000000000000000000000000d" +_QUIET = __import__("logging").getLogger("test.probe.fake-upstream") +_QUIET.propagate = False +_QUIET.setLevel(__import__("logging").CRITICAL + 1) + + +async def _upstream(process_request, handler=None): + """A websockets server on a free loopback port. A refusal must come from + ``connection.respond()`` in ``process_request`` (websockets 13 runs the + handler for a hand-built Response).""" + from websockets.asyncio.server import serve + + async def echo(ws): + async for msg in ws: + await ws.send(msg) + + server = await serve( + handler or echo, + "127.0.0.1", + 0, + process_request=process_request, + logger=_QUIET, + ) + return server, next(iter(server.sockets)).getsockname()[1] + + +def _proxy_to(port): + proxy = ProbeProxy( + api_key=KEY, + upstream_base=f"http://127.0.0.1:{port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) + assert proxy.upstream_base == f"ws://127.0.0.1:{port}" + return proxy + + +async def _client_until_closed(port, target="/v1/listen?encoding=linear16"): + from websockets.asyncio.client import connect + from websockets.exceptions import ConnectionClosed, InvalidHandshake + + try: + async with connect(f"ws://127.0.0.1:{port}{target}", open_timeout=5) as ws: + await asyncio.wait_for(ws.recv(), 10) + except (ConnectionClosed, InvalidHandshake, OSError, asyncio.TimeoutError): + pass + + +def _run_proxy_against(process_request, *, connections=1): + """Serve the fake upstream and the proxy on loopback, connect + ``connections`` clients one after another, and run until the proxy stops + itself or the clients are done. Returns (proxy, seen upstream auth).""" + seen = [] + + def record(connection, request): + seen.append(request.headers.get("Authorization")) + return process_request(connection, request) + + async def main(): + upstream, up_port = await _upstream(record) + proxy = _proxy_to(up_port) + port = _free_port() + run = asyncio.ensure_future(proxy.run(port)) + await asyncio.sleep(0.1) + for _ in range(connections): + await _client_until_closed(port) + await asyncio.sleep(0.3) + proxy._stop.set() + await asyncio.wait_for(run, 20) + upstream.close() + await upstream.wait_closed() + return proxy + + return asyncio.run(main()), seen + + +def _refusing(status): + def process_request(connection, request): + return connection.respond(status, "{}") + + return process_request + + +@pytest.mark.parametrize(("status", "code"), [(401, "auth_invalid")]) +def test_refused_key_stops_the_proxy_with_its_code(status, code): + proxy, seen = _run_proxy_against(_refusing(status), connections=1) + assert proxy.fatal is not None + assert proxy.fatal.code == code + assert seen == [f"Token {KEY}"] + assert KEY not in proxy.fatal.error + + +@pytest.mark.parametrize("status", [400, 403, 404, 429, 500, 503]) +def test_other_refusals_fail_only_that_connection(status, capsys): + proxy, seen = _run_proxy_against(_refusing(status), connections=2) + assert proxy.fatal is None # the proxy kept serving + assert seen == [f"Token {KEY}"] * 2 + assert len(proxy.connections) == 2 + err = capsys.readouterr().err + assert err.count("Deepgram connection failed") == 2 + assert KEY not in err + if status == 404: + assert "/v1/listen" in err + + +def test_a_later_401_doesnt_stop_the_loop_again(): + """A second client refused after the first fatal 401 (here: while the + server shuts down) must not stop the proxy a second time.""" + proxy, seen = _run_proxy_against(_refusing(401), connections=2) + assert proxy.fatal is not None and proxy.fatal.code == "auth_invalid" + assert 1 <= len(seen) <= 2 + + +def test_fatal_stop_with_an_open_stream_is_fast(capsys): + """A 401 while another stream is open stops within seconds, with + no leftover tasks or loop exception-handler errors.""" + import time + + def process_request(connection, request): + if "mode=401" in request.path: + return connection.respond(401, "{}") + return None + + errors: list[dict] = [] + marks: dict[str, float] = {} + + async def main(): + asyncio.get_running_loop().set_exception_handler( + lambda _loop, ctx: errors.append(ctx) + ) + upstream, up_port = await _upstream(process_request) + proxy = _proxy_to(up_port) + port = _free_port() + run = asyncio.ensure_future(proxy.run(port)) + await asyncio.sleep(0.1) + + async def held(): + from websockets.asyncio.client import connect + from websockets.exceptions import ConnectionClosed + + async with connect(f"ws://127.0.0.1:{port}/v1/listen") as ws: + await ws.send("ping") + await asyncio.wait_for(ws.recv(), 5) + try: + await asyncio.wait_for(ws.recv(), 30) # until the proxy closes it + except ConnectionClosed: + pass + + holder = asyncio.ensure_future(held()) + await asyncio.sleep(0.5) + marks["refused"] = time.monotonic() + await _client_until_closed(port, "/v1/listen?mode=401") + await asyncio.wait_for(run, 30) + marks["stopped"] = time.monotonic() + await asyncio.wait_for(holder, 5) + upstream.close() + await upstream.wait_closed() + return proxy + + proxy = asyncio.run(main()) + assert proxy.fatal is not None and proxy.fatal.code == "auth_invalid" + assert marks["stopped"] - marks["refused"] < 10 + assert errors == [] + + +def test_open_timeout_fails_only_that_connection(monkeypatch, capsys): + from deepctl_core import transport + + monkeypatch.setattr(transport, "WS_OPEN_TIMEOUT_S", 0.3) + + async def hang(connection, request): + await asyncio.sleep(2) + + proxy, seen = _run_proxy_against(hang, connections=1) + assert proxy.fatal is None + assert "(timeout)" in capsys.readouterr().err + + +def test_default_upstream_is_the_hosts_default(): + from deepctl_core.hosts import DEFAULT_BASE_URL + + proxy = ProbeProxy(api_key=KEY) + assert proxy.upstream_base == DEFAULT_BASE_URL.replace("https://", "wss://") + assert proxy.upstream_headers == {} # no key without the command's check + + +def test_refused_host_gets_no_key(monkeypatch, tmp_path): + from deepctl_core import hosts, output + from deepctl_core.auth import AuthManager + from deepctl_core.config import Config + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setattr(hosts, "_reported", set()) + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv("DEEPGRAM_API_KEY", KEY) + config = Config(config_path=str(tmp_path / "none.yaml")) + built = [] + # handle() imports ProbeProxy from .proxy at call time, so the + # seam is the proxy module's name. + monkeypatch.setattr( + "deepctl_cmd_debug_probe.proxy.ProbeProxy", + lambda **kw: built.append(kw), + ) + with patch("deepctl_cmd_debug_probe.command.require_ffprobe", return_value=True): + with pytest.raises(hosts.CredentialHostError): + ProbeCommand().handle(config, AuthManager(config), Mock(), port=None) + assert built == [] diff --git a/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py b/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py index 43eeda4b..f23d9e98 100644 --- a/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py +++ b/packages/deepctl-cmd-debug-probe/tests/unit/test_proxy_loopback.py @@ -1,7 +1,8 @@ """The probe proxy against real loopback servers: relay, refusals, run loop. The upstream is a websockets server on a free 127.0.0.1 port, reached -through the proxy's `_upstream_base` seam. Nothing is mocked. +through the proxy's `upstream_base`, with the Authorization the command's +Transport would pass. Nothing is mocked. """ from __future__ import annotations @@ -81,8 +82,12 @@ class Proxy: """A ProbeProxy listening on a free port, pointed at `upstream`.""" def __init__(self, upstream: FakeUpstream) -> None: - self.proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=10**9) - self.proxy._upstream_base = f"ws://127.0.0.1:{upstream.port}" + self.proxy = ProbeProxy( + api_key=KEY, + probe_interval_bytes=10**9, + upstream_base=f"ws://127.0.0.1:{upstream.port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) async def __aenter__(self) -> Proxy: self.server = await self.proxy.start(0) @@ -220,7 +225,7 @@ async def main() -> tuple[int, bytes, FakeUpstream]: status, body, up = asyncio.run(main()) assert (status, body) == (403, b"Forbidden.\n") assert up.handshakes == [] - out = " ".join(capsys.readouterr().out.split()) # rich wraps at 80 columns + out = " ".join(capsys.readouterr().err.split()) # rich wraps at 80 columns if "Origin" in headers: shown = headers["Origin"].replace("\t", "\\x09") assert ( @@ -269,15 +274,17 @@ async def main() -> None: await ws.close() asyncio.run(main()) - assert "path=/v1/listen[bold]x" in capsys.readouterr().out + assert "path=/v1/listen[bold]x" in capsys.readouterr().err -def test_upstream_401_reports_and_keeps_serving(capsys, caplog): +def test_upstream_403_reports_and_keeps_serving(capsys, caplog): + """A refusal other than 401 fails only that connection (only + auth_invalid stops the proxy; see test_probe's fatal tests).""" caplog.set_level(logging.DEBUG) async def main() -> list[tuple[int | None, str | None]]: closes = [] - async with FakeUpstream(status=401) as up, Proxy(up) as px: + async with FakeUpstream(status=403) as up, Proxy(up) as px: for _ in range(2): async with connect(f"ws://127.0.0.1:{px.port}/v1/listen") as ws: with pytest.raises(ConnectionClosed): @@ -288,8 +295,8 @@ async def main() -> list[tuple[int | None, str | None]]: assert asyncio.run(main()) == [(1000, ""), (1000, "")] out = capsys.readouterr() - assert out.out.count("Upstream connection failed") == 2 - assert "HTTP 401" in out.out + assert out.err.count("Deepgram connection failed") == 2 + assert "(auth_forbidden)" in out.err for text in (out.out, out.err, caplog.text): assert KEY not in text @@ -331,7 +338,7 @@ def test_run_stops_on_event_and_on_cancel_after_summaries(capsys): async def main() -> ProbeProxy: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" with __import__("socket").socket() as s: s.bind(("127.0.0.1", 0)) port = s.getsockname()[1] @@ -361,7 +368,7 @@ async def main() -> ProbeProxy: proxy = asyncio.run(main()) (stats,) = proxy.connections assert stats.duration_seconds is not None - out = capsys.readouterr().out + out = capsys.readouterr().err assert "Connection closed" in out assert "Sent: 1,000 bytes (1 binary, 0 text frames)" in out @@ -554,14 +561,14 @@ def test_snapshots_do_not_block_the_loop_and_stop_waits_for_summaries( proxymod, "get_ffprobe_path", lambda _config=None: _fake_ffprobe( - tmp_path, f"open({str(marker)!r}, 'w').close(); time.sleep(1.5)" + tmp_path, f"open({str(marker)!r}, 'w').close(); time.sleep(4)" ), ) async def main() -> tuple[float, ProbeProxy]: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=1) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -570,7 +577,7 @@ async def main() -> tuple[float, ProbeProxy]: await asyncio.sleep(0.2) a = await connect(f"ws://127.0.0.1:{port}/v1/listen") await a.recv() - await a.send(b"\x00" * 100) # starts a 1.5 s snapshot + await a.send(b"\x00" * 100) # starts a 4 s snapshot while not marker.exists(): await asyncio.sleep(0.01) t0 = time.monotonic() @@ -579,16 +586,16 @@ async def main() -> tuple[float, ProbeProxy]: await b.send("ping") assert await b.recv() == "ping" relayed_in = time.monotonic() - t0 - for _ in range(30): # a backlog: 30 more 1.5 s snapshots if drained + for _ in range(30): # a backlog: 30 more 4 s snapshots if drained await a.send(b"\x00" * 100) proxy._stop.set() # what the first Ctrl+C does, mid-snapshot - await asyncio.wait_for(task, 10) + await asyncio.wait_for(task, 20) return relayed_in, proxy relayed_in, proxy = asyncio.run(main()) - assert relayed_in < 1.0 + assert relayed_in < 2.5 assert all(c.duration_seconds is not None for c in proxy.connections) - out = capsys.readouterr().out + out = capsys.readouterr().err assert out.count("Connection closed") == 2 assert "Not forwarded (arrived while stopping): 3,000 bytes (30 frames)" in out @@ -601,7 +608,7 @@ def test_abort_ends_connections_without_waiting_for_close_timeout(capsys): async def main() -> float: async with _DeafUpstream() as up: proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -619,8 +626,8 @@ async def main() -> float: await asyncio.wait_for(task, 10) return time.monotonic() - t0 - assert asyncio.run(main()) < 3 - assert "Connection closed" in capsys.readouterr().out + assert asyncio.run(main()) < 10 + assert "Connection closed" in capsys.readouterr().err def test_abort_during_upstream_handshake(): @@ -631,7 +638,7 @@ async def never(reader: Any, writer: Any) -> None: hang = await asyncio.start_server(never, "127.0.0.1", 0) proxy = ProbeProxy(api_key=KEY) - proxy._upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" + proxy.upstream_base = f"ws://127.0.0.1:{hang.sockets[0].getsockname()[1]}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -675,7 +682,7 @@ def test_second_stop_kills_a_running_ffprobe_and_skips_the_final_probe( async def main() -> float: async with FakeUpstream() as up: proxy = ProbeProxy(api_key=KEY, probe_interval_bytes=1) - proxy._upstream_base = f"ws://127.0.0.1:{up.port}" + proxy.upstream_base = f"ws://127.0.0.1:{up.port}" server = await proxy.start(0) port = next(iter(server.sockets)).getsockname()[1] server.close() @@ -695,9 +702,9 @@ async def main() -> float: return time.monotonic() - t0 elapsed = asyncio.run(main()) - assert elapsed < 1.5 + assert elapsed < 4.0 assert len(list(runs.iterdir())) == 1 # no final probe - assert "Connection closed" in capsys.readouterr().out + assert "Connection closed" in capsys.readouterr().err def test_transcripts_print_literally_and_keep_unicode(capsys): @@ -714,7 +721,7 @@ def test_transcripts_print_literally_and_keep_unicode(capsys): ), stats, ) - out = capsys.readouterr().out + out = capsys.readouterr().err assert "[bold]x[/bold] :smile: \u3053\u3093\u306b\u3061\u306f[2J" in out assert "\x1b" not in out and "\x9b" not in out assert stats.transcripts == [transcript] @@ -732,6 +739,85 @@ def test_not_forwarded_line_counts(frames, size, line, capsys): stats = ProbeConnectionStats(connection_id="1234abcd") ProbeProxy(api_key=KEY)._print_connection_summary(stats, frames, size) - assert f" {line}\n" in capsys.readouterr().out + assert f" {line}\n" in capsys.readouterr().err ProbeProxy(api_key=KEY)._print_connection_summary(stats) - assert "Not forwarded" not in capsys.readouterr().out + assert "Not forwarded" not in capsys.readouterr().err + + +class _Redirector: + """A loopback upstream that answers every upgrade with a redirect and + counts the handshakes it gets and the Authorization values it saw.""" + + def __init__(self, status: int, location: str) -> None: + self.status = status + self.location = location + self.handshakes: list[str | None] = [] + + def _process(self, connection, request): # noqa: ANN001 + self.handshakes.append(request.headers.get("Authorization")) + response = connection.respond(self.status, "moved\n") + response.headers["Location"] = self.location.format(port=self.port) + return response + + async def __aenter__(self) -> "_Redirector": + from websockets.asyncio.server import serve + + self.server = await serve( + lambda ws: ws.close(), "127.0.0.1", 0, process_request=self._process + ) + self.port = self.server.sockets[0].getsockname()[1] + return self + + async def __aexit__(self, *exc: object) -> None: + self.server.close() + await self.server.wait_closed() + + +@pytest.mark.parametrize( + ("status", "cross_origin"), + [(302, True), (302, False), (301, True), (308, True), (307, False)], +) +def test_upstream_redirect_is_never_followed(status, cross_origin, capsys): + """A 3xx on the upstream upgrade fails only that connection with + network_unavailable; the redirect target sees no connection and the key + is sent once, to the configured upstream only.""" + + async def main(): + async with FakeUpstream() as target: + location = ( + f"ws://127.0.0.1:{target.port}/v1/listen" + if cross_origin + else "ws://127.0.0.1:{port}/v1/listen?again=1" + ) + async with _Redirector(status, location) as up: + proxy = ProbeProxy( + api_key=KEY, + upstream_base=f"ws://127.0.0.1:{up.port}", + upstream_headers={"Authorization": f"Token {KEY}"}, + ) + server = await proxy.start(0) + port = server.sockets[0].getsockname()[1] + err = "" + try: + await _status(port, _upgrade(port)) + for _ in range(150): + err += capsys.readouterr().err + if "connection failed" in err: + break + await asyncio.sleep(0.02) + # A followed redirect would reach the target meanwhile. + await asyncio.sleep(0.2) + err += capsys.readouterr().err + finally: + server.close() + await server.wait_closed() + return err, up.handshakes, target.handshakes, proxy + + err, handshakes, target_handshakes, proxy = asyncio.run(main()) + assert handshakes == [f"Token {KEY}"] + assert target_handshakes == [] + assert proxy.fatal is None + flat = " ".join(err.split()) + assert "Deepgram connection failed" in flat + assert "(network_unavailable)" in flat + assert KEY not in err diff --git a/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py b/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py index 1e2a69c3..0e47f501 100644 --- a/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py +++ b/packages/deepctl-cmd-debug-toolkit/src/deepctl_cmd_debug_toolkit/command.py @@ -9,6 +9,9 @@ import click from deepctl_core import AuthenticationError, AuthManager, BaseGroupCommand, Config + +# Already loaded with BaseGroupCommand (deepctl_core.base_command imports it). +from deepctl_core.hosts import DEFAULT_BASE_URL from rich.console import Console from .fetcher import get_cached_manifest, get_or_fetch_script, refresh_manifest @@ -16,6 +19,9 @@ console = Console() +# The streaming endpoint the examples point at, built from the one default. +_LISTEN_WS_URL = DEFAULT_BASE_URL.replace("https://", "wss://", 1) + "/v1/listen" + class ToolkitCommand(BaseGroupCommand): """Runs field engineering scripts from deepgram/support-toolkit.""" @@ -35,8 +41,8 @@ class ToolkitCommand(BaseGroupCommand): examples = [ "dg debug toolkit refresh", - "dg debug toolkit latency wss://api.deepgram.com/v1/listen", - "dg debug toolkit latency wss://api.deepgram.com/v1/listen --duration=60", + f"dg debug toolkit latency {_LISTEN_WS_URL}", + f"dg debug toolkit latency {_LISTEN_WS_URL} --duration=60", ] def setup_commands(self) -> list[click.Command]: diff --git a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py index 93c7e3ba..cf91ff11 100644 --- a/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py +++ b/packages/deepctl-cmd-listen/src/deepctl_cmd_listen/command.py @@ -101,6 +101,14 @@ def _ws_base(client: DeepgramClient) -> str: return f"{'ws' if scheme.lower() in ('http', 'ws') else 'wss'}://{rest}" +def _socket_headers(api_key: str | None) -> dict[str, str]: + """What a live stream's WebSocket open carries: the transport's request + headers. The host was checked when the URL was built (``_ws_base``).""" + from deepctl_core import transport + + return transport.request_headers(api_key or "") + + # asyncio (and with it ssl) loads only when a live stream starts, so # `dg listen --help` and the prerecorded path don't import it. # The live engine imports asyncio, so it loads lazily too. @@ -852,7 +860,16 @@ def _prerecorded( # Raise (not return an error result) so an API rejection — e.g. a # --redact value the v1 endpoint refuses — exits non-zero and is # visible, rather than printing nothing and exiting 0. - detail = scrub_url_text(e, source) if url_like else e + from deepctl_core.output import print_debug + from deepctl_core.transport import plain_detail + + # The cause in its own words, never the SDK error's field dump; + # the whole exception only with --verbose. + raw = plain_detail(e) + detail = scrub_url_text(raw, source) if url_like else raw + print_debug( + escape(scrub_url_text(repr(e), source) if url_like else repr(e)) + ) raise click.ClickException(f"Transcription failed: {detail}") # ── Format transcript ────────────────────────────────────────── @@ -1081,7 +1098,7 @@ def on_message(raw: str | bytes) -> None: try: outcome = await live.run_session( url, - {"Authorization": f"Token {api_key}"}, + _socket_headers(api_key), on_message=on_message, queue=queue, closing=closing, @@ -1263,7 +1280,7 @@ def on_message(raw: str | bytes) -> None: outcome = await live.run_session( url, - {"Authorization": f"Token {api_key}"}, + _socket_headers(api_key), on_message=on_message, queue=queue, closing=closing, diff --git a/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py b/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py index db905f42..3ff5ac3e 100644 --- a/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py +++ b/packages/deepctl-cmd-listen/tests/unit/test_live_redirect.py @@ -14,9 +14,9 @@ from dataclasses import dataclass, field from unittest.mock import MagicMock -import click import pytest from deepctl_cmd_listen.command import ListenCommand +from deepctl_core.websocket import RedirectRefused FAKE_KEY = "fa1ce0000000000000000000000000000000000d" @@ -157,7 +157,7 @@ def test_cross_origin_redirect_is_refused(mode, no_audio_devices): assert f"Token {FAKE_KEY}".encode() in servers.origin.requests[0] # Nothing at all reached the redirect target. assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.exit_code != 0 target = f"ws://127.0.0.1:{servers.target.port}" assert err.message == ( @@ -174,7 +174,7 @@ def test_same_origin_redirect_is_refused(mode, no_audio_devices): # The redirect back to the same host is not followed either. assert len(servers.origin.requests) == 1 assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.exit_code != 0 assert "doesn't follow redirects" in err.message @@ -185,7 +185,7 @@ def test_relative_location_names_the_same_origin(no_audio_devices): origin = f"ws://127.0.0.1:{servers.origin.port}" assert len(servers.origin.requests) == 1 - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message.startswith(f"{origin} answered with a redirect to {origin}.") @@ -194,7 +194,7 @@ def test_control_bytes_in_location_are_not_echoed(no_audio_devices): with _Servers("ws://evil\x9b31m.example/x") as servers: err = _run("stdin", servers.origin.origin) - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message.isascii() assert all(c.isprintable() for c in err.message) assert "redirect to ws://evil. " in err.message @@ -210,8 +210,22 @@ def test_unbalanced_bracket_in_location_gives_the_redirect_message( err = _run(mode, servers.origin.origin) assert servers.target.requests == [] - assert isinstance(err, click.ClickException) + assert isinstance(err, RedirectRefused) assert err.message == ( f"ws://127.0.0.1:{servers.origin.port} answered with a redirect to ?. " "dg listen doesn't follow redirects, so no audio was sent." ) + + +@pytest.mark.parametrize("mode", ["mic", "stdin"]) +def test_upgrade_carries_the_request_headers(mode, no_audio_devices): + """The live upgrade identifies the run as every other API request does: + the SDK's User-Agent, the session ID and the key.""" + from deepctl_core import transport + + with _Servers(CROSS) as servers: + _run(mode, servers.origin.origin) + + request = servers.origin.requests[0].decode("latin-1").lower() + for name, value in transport.request_headers(FAKE_KEY).items(): + assert f"\r\n{name.lower()}: {value.lower()}\r\n" in request, name diff --git a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py index 32a369e5..24d26abd 100644 --- a/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py +++ b/packages/deepctl-cmd-login/src/deepctl_cmd_login/command.py @@ -1531,12 +1531,10 @@ def handle( # Try to get API key from keyring to verify access try: - import keyring - from deepctl_core.auth import KEYRING_SERVICE + from deepctl_core import credentials - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{switch_profile}" - ) + # Bounded off a terminal (2 s); no answer counts as no entry. + api_key = credentials.keyring_get(f"api-key.{switch_profile}") if not api_key: # Check if API key is in config instead @@ -1625,48 +1623,23 @@ def handle( **kwargs: Any, ) -> Any: """Handle whoami command.""" - import os - - import keyring as _keyring - from deepctl_core.auth import KEYRING_SERVICE - if kwargs.get("profile"): config.profile = kwargs["profile"] profile_name = config.profile or "default" profile_cfg = config.get_profile(profile_name) - # Determine key and its exact storage source. Expired finite profile - # credentials must not mask a valid environment variable. - api_key: str | None = None - key_source = "not set" - env_key = os.environ.get("DEEPGRAM_API_KEY") - has_profile_key, _ = auth_manager.has_profile_credentials(profile_name) - - if has_profile_key: - try: - api_key = _keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) - if api_key: - key_source = "system keyring" - except Exception: - pass - - if has_profile_key and not api_key and profile_cfg.api_key: - # Config merges DEEPGRAM_API_KEY into the profile at load time - # (environment overrides the file), so a profile key equal to the - # environment value came from the environment, not the config - # file — label it accordingly instead of claiming "config file". - api_key = profile_cfg.api_key - if env_key and profile_cfg.api_key == env_key: - key_source = "DEEPGRAM_API_KEY (env)" - else: - key_source = "config file" - - if not api_key and env_key: - api_key = env_key - key_source = "DEEPGRAM_API_KEY (env)" + # The key and its exact storage source, from the one resolver (its + # key_source uses these same words): one bounded keyring read, and an + # expired finite profile credential doesn't mask the environment's. + # Like before, --api-key isn't a storage source here. Read-only, as in + # 0.4.2: an expired credential's keyring entry stays (prune=False). + resolved = auth_manager.resolve( + profile=profile_name, use_flags=False, prune=False + ) + api_key = resolved.api_key + key_source = resolved.key_source + has_profile_key = resolved.profile_has_key authenticated = auth_manager.is_authenticated() masked: str | None = None @@ -1674,7 +1647,9 @@ def handle( masked = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****" project_id = auth_manager.get_project_id() - base_url = profile_cfg.base_url or "https://api.deepgram.com" + from deepctl_core.hosts import DEFAULT_BASE_URL + + base_url = profile_cfg.base_url or DEFAULT_BASE_URL # Render the human block only in default mode. For json/yaml/csv the # framework serialises the returned result to stdout, so printing the @@ -1693,7 +1668,7 @@ def handle( if project_id else " Project ID: [dim]not set[/dim]" ) - if base_url != "https://api.deepgram.com": + if base_url != DEFAULT_BASE_URL: console.print(f" Base URL: {base_url}") if profile_cfg.dg_token_expires_at and has_profile_key: console.print(f" Expires: {profile_cfg.dg_token_expires_at}") diff --git a/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py b/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py new file mode 100644 index 00000000..790c1ddb --- /dev/null +++ b/packages/deepctl-cmd-login/tests/unit/test_keyring_bound.py @@ -0,0 +1,146 @@ +"""Whoami, profiles --switch and profiles --list read the keyring +through the resolver's bound (2 s off a terminal), not directly.""" + +from __future__ import annotations + +import os +import sys +import threading +import time +from unittest.mock import MagicMock + +import pytest +import yaml +from deepctl_core import auth as auth_module +from deepctl_core import config as config_module +from deepctl_core import credentials, hosts, output +from deepctl_core.auth import AuthManager +from deepctl_core.config import Config + +pytestmark = pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="XDG layout (Linux)" +) + + +class HangingKeyring: + def __init__(self) -> None: + self.release = threading.Event() + self.reads = 0 + + def get_password(self, service: str, username: str) -> str | None: + self.reads += 1 + self.release.wait(10) + return "k" * 40 + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + monkeypatch.setattr(hosts, "_reported", set()) + home = tmp_path / "home" + (home / ".config" / "deepctl").mkdir(parents=True) + cfg = home / ".config" / "deepctl" / "config.yaml" + cfg.write_text( + yaml.safe_dump( + {"profiles": {"default": {"project_id": "p"}, "work": {"api_key": "w" * 40}}} + ) + ) + os.chmod(cfg, 0o600) + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setattr(credentials, "KEYRING_BOUND_S", 0.3) + monkeypatch.setattr(credentials, "_interactive", lambda: False) + kr = HangingKeyring() + monkeypatch.setattr(auth_module, "keyring", kr, raising=False) + yield kr + kr.release.set() + + +def _timed(fn): + started = time.monotonic() + result = fn() + return result, time.monotonic() - started + + +def test_whoami_is_bounded(setup, capsys): + from deepctl_cmd_login.command import WhoamiCommand + + config = Config() + result, took = _timed( + lambda: WhoamiCommand().handle(config, AuthManager(config), MagicMock()) + ) + assert took < 5.0 # the keyring hangs until the test ends + assert result.key_source == "DEEPGRAM_API_KEY (env)" + assert result.authenticated is True + assert setup.reads == 1 # one bounded read, then cached + assert capsys.readouterr().err.count("keyring didn't answer") == 1 + + +def test_profiles_switch_is_bounded(setup, monkeypatch): + from deepctl_cmd_login.command import ProfilesCommand + + config = Config() + monkeypatch.setattr(config, "save", lambda *a, **k: None) + result, took = _timed( + lambda: ProfilesCommand().handle( + config, AuthManager(config), MagicMock(), switch="work" + ) + ) + assert took < 5.0 # the keyring hangs until the test ends + assert result.status != "error" # fell back to the profile's file key + + +def test_profiles_list_is_bounded(setup): + config = Config() + result, took = _timed(lambda: AuthManager(config).list_profiles()) + assert took < 5.0 # the keyring hangs until the test ends + # A timeout falls back to the file, like a keyring error did in 0.4.2. + assert result.profiles["work"].api_key == "****" + "w" * 4 + + +def test_whoami_leaves_an_expired_entry_alone(tmp_path, monkeypatch): + """Whoami is read-only, as in 0.4.2; key_source unchanged.""" + from deepctl_cmd_login.command import WhoamiCommand + + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + home = tmp_path / "home" + (home / ".config" / "deepctl").mkdir(parents=True) + cfg = home / ".config" / "deepctl" / "config.yaml" + cfg.write_text( + yaml.safe_dump( + {"profiles": {"default": {"dg_token_expires_at": "2020-01-01T00:00:00+00:00"}}} + ) + ) + os.chmod(cfg, 0o600) + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + + class Recording: + def __init__(self) -> None: + self.entries = {"api-key.default": "k" * 40} + self.calls: list[str] = [] + + def get_password(self, service, username): + self.calls.append("get") + return self.entries.get(username) + + def delete_password(self, service, username): + self.calls.append("delete") + self.entries.pop(username, None) + + kr = Recording() + monkeypatch.setattr(auth_module, "keyring", kr, raising=False) + config = Config() + result = WhoamiCommand().handle(config, AuthManager(config), MagicMock()) + assert result.key_source == "DEEPGRAM_API_KEY (env)" + assert "delete" not in kr.calls + assert kr.entries == {"api-key.default": "k" * 40} diff --git a/packages/deepctl-cmd-login/tests/unit/test_login_command.py b/packages/deepctl-cmd-login/tests/unit/test_login_command.py index e63fa16d..f487a58e 100644 --- a/packages/deepctl-cmd-login/tests/unit/test_login_command.py +++ b/packages/deepctl-cmd-login/tests/unit/test_login_command.py @@ -552,16 +552,18 @@ def whoami_command(self): return WhoamiCommand() def _run(self, whoami_command, mock_config, mock_client, env, profile_key): - auth_manager = Mock(spec=AuthManager) - auth_manager.get_project_id.return_value = "proj-1" - auth_manager.has_profile_credentials.return_value = (bool(profile_key), False) - auth_manager.is_authenticated.return_value = bool(profile_key or env) + # Whoami takes the key and its source from the resolver, so + # this runs the real AuthManager over the mocked config and keyring. mock_config.get_profile.return_value.api_key = profile_key + mock_config.get_profile.return_value.project_id = "proj-1" mock_config.get_profile.return_value.dg_token_expires_at = None + keyring = Mock() + keyring.get_password.return_value = None with ( patch.dict("os.environ", env, clear=False), - patch("keyring.get_password", return_value=None), + patch("deepctl_core.auth.keyring", keyring, create=True), ): + auth_manager = AuthManager(mock_config) if "DEEPGRAM_API_KEY" not in env: import os diff --git a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py index 8d153221..35deb9c2 100644 --- a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py +++ b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py @@ -1,7 +1,11 @@ """First-run prompt to nudge users to set up AI assistant skills. -Checks for installed AI coding CLIs and prints a one-line notification -to stderr. Shows only once (uses a cache file to track). +Checks for installed AI coding CLIs in a daemon thread and prints a +one-line notification to stderr after the command, only if the thread has +already finished; dg never waits for it. Shows only once (a cache file +records that it was shown), and only to a person at a terminal: never in +CI, an AI agent session, under ``dg mcp``, with ``DEEPCTL_NO_UPDATE_CHECK`` +set, or when stderr isn't a terminal. """ from __future__ import annotations @@ -41,13 +45,44 @@ def _is_oneshot() -> bool: return bool(exe_path.startswith(pipx_cache)) +def _stderr_is_terminal() -> bool: + try: + return bool(sys.stderr is not None and sys.stderr.isatty()) + except (AttributeError, OSError, ValueError): + return False + + +def _is_agent() -> bool: + try: + from deepctl_core.output import is_agentic, is_ai_agent + except ImportError: + return False + return is_agentic() or is_ai_agent() + + +def _opted_out() -> bool: + """``DEEPCTL_NO_UPDATE_CHECK`` turns off every startup check.""" + value = os.environ.get("DEEPCTL_NO_UPDATE_CHECK", "").strip().lower() + return value not in ("", "0", "false", "no") + + +def _is_mcp() -> bool: + """This run is ``dg mcp`` (root options before it allowed).""" + try: + from deepctl_cmd_update.startup_check import _subcommand + except ImportError: + args = [a for a in sys.argv[1:] if not a.startswith("-")] + return bool(args) and args[0] == "mcp" + return _subcommand() == "mcp" + + def _already_prompted() -> bool: return _CACHE_FILE.exists() def _mark_prompted() -> None: try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) + _CACHE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) _CACHE_FILE.write_text("1") except OSError: pass @@ -66,7 +101,6 @@ def _background_detect(result: dict[str, bool]) -> None: detected = detect_ai_clis() if detected: result["should_prompt"] = True - _mark_prompted() except Exception: pass @@ -83,29 +117,42 @@ def check_and_notify(quiet: bool = False) -> None: after CLI execution to display the result. """ global _thread, _result + _thread = None _result = {} - if quiet or _is_ci() or _is_oneshot() or _already_prompted(): + if ( + quiet + or _opted_out() + or _is_ci() + or not _stderr_is_terminal() + or _is_agent() + or _is_oneshot() + or _is_mcp() + or _already_prompted() + ): return _thread = threading.Thread( target=_background_detect, args=(_result,), + name="deepctl-skills-check", daemon=True, ) _thread.start() def print_pending_notification() -> None: - """Print the skills prompt if AI CLIs were detected.""" + """Print the skills prompt if the check already found AI CLIs. + + Never waits: a check that is still running is dropped, and the prompt + is recorded as shown only once it is printed. + """ global _thread - if _thread is None: + thread, _thread = _thread, None + if thread is None or thread.is_alive(): return - _thread.join(timeout=2.0) - _thread = None - if _result.get("should_prompt"): try: sys.stderr.write( @@ -115,4 +162,5 @@ def print_pending_notification() -> None: except (BrokenPipeError, OSError, ValueError): # The output stream closed (e.g. an MCP host disconnected stdio # after `dg mcp` finished). The prompt is purely advisory. - pass + return + _mark_prompted() diff --git a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py index 700bc305..4567d8e3 100644 --- a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py +++ b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py @@ -965,6 +965,104 @@ def test_suppressed_when_quiet(self): startup_check.check_and_notify(quiet=True) assert startup_check._thread is None + def test_suppressed_when_stderr_is_not_a_terminal(self): + from deepctl_cmd_skills import startup_check + + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=False), + patch.object(startup_check, "_is_oneshot", return_value=False), + patch.object(startup_check, "_already_prompted", return_value=False), + patch.object(startup_check, "_stderr_is_terminal", return_value=False), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + @pytest.mark.parametrize("case", ["mcp", "opt-out"]) + def test_suppressed_for_mcp_and_opt_out(self, case, monkeypatch): + from deepctl_cmd_skills import startup_check + + monkeypatch.delenv("DEEPCTL_NO_UPDATE_CHECK", raising=False) + if case == "opt-out": + monkeypatch.setenv("DEEPCTL_NO_UPDATE_CHECK", "1") + mcp = False + else: + mcp = True + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=False), + patch.object(startup_check, "_is_oneshot", return_value=False), + patch.object(startup_check, "_already_prompted", return_value=False), + patch.object(startup_check, "_stderr_is_terminal", return_value=True), + patch.object(startup_check, "_is_mcp", return_value=mcp), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + def test_mcp_detected_through_root_options(self, monkeypatch): + from deepctl import _cli + from deepctl_cmd_skills import startup_check + + monkeypatch.setattr(_cli, "_run_args", ["--profile", "x", "mcp"]) + assert startup_check._is_mcp() is True + monkeypatch.setattr(_cli, "_run_args", ["models"]) + assert startup_check._is_mcp() is False + + def test_suppressed_for_an_agent(self): + from deepctl_cmd_skills import startup_check + + with ( + patch.object(startup_check, "_is_ci", return_value=False), + patch.object(startup_check, "_is_agent", return_value=True), + patch.object(startup_check, "_stderr_is_terminal", return_value=True), + patch.object(startup_check.threading, "Thread") as thread_cls, + ): + startup_check.check_and_notify(quiet=False) + thread_cls.assert_not_called() + + def test_running_check_is_dropped_without_waiting(self): + import threading + + from deepctl_cmd_skills import startup_check + + release = threading.Event() + startup_check._result = {"should_prompt": True} + thread = threading.Thread(target=release.wait, daemon=True) + thread.start() + startup_check._thread = thread + try: + with ( + patch.object(thread, "join") as join, + patch.object(startup_check, "_mark_prompted") as mark, + patch.object(sys, "stderr", MagicMock()) as err, + ): + startup_check.print_pending_notification() + join.assert_not_called() + err.write.assert_not_called() + mark.assert_not_called() + finally: + release.set() + thread.join(timeout=5.0) + + def test_marked_only_once_shown(self): + import threading + + from deepctl_cmd_skills import startup_check + + startup_check._result = {"should_prompt": True} + startup_check._thread = threading.Thread(target=lambda: None) + startup_check._thread.start() + startup_check._thread.join() + with ( + patch.object(startup_check, "_mark_prompted") as mark, + patch.object(sys, "stderr", MagicMock()) as err, + ): + startup_check.print_pending_notification() + err.write.assert_called_once() + mark.assert_called_once() + @pytest.mark.parametrize( "error", [ diff --git a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py index be8da794..cc547559 100644 --- a/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py +++ b/packages/deepctl-cmd-speak/src/deepctl_cmd_speak/command.py @@ -294,6 +294,19 @@ def _os_error_text(exc: OSError) -> str: _CLOSE_TIMEOUT_S = 2.0 +def _failed(what: str, exc: BaseException) -> click.ClickException: + """``{what}: {cause}`` for an error no coded message covers. + + The cause is the underlying error's own message, never the SDK error's + field dump; the whole exception goes to --verbose output only. + """ + from deepctl_core.output import print_debug + from deepctl_core.transport import plain_detail + + print_debug(escape(f"{what}: {exc!r}")) + return click.ClickException(f"{what}: {plain_detail(exc)}") + + def _close(*streams: object, timeout: float = _CLOSE_TIMEOUT_S) -> None: """Close audio iterators (and the responses behind them), quietly and in order, and wait at most ``timeout`` seconds for all of them together.""" @@ -1431,7 +1444,7 @@ def _speak_flux( except _SINK_ERRORS: raise except Exception as e: - raise click.ClickException(f"Flux streaming failed: {e}") + raise _failed("Flux streaming failed", e) from e finally: _close(stream, raw) @@ -1609,7 +1622,7 @@ def _speak_v1( sample_rate=sample_rate, ) except Exception as e: - raise click.ClickException(f"Error generating speech: {e}") + raise _failed("Error generating speech", e) from e try: if player in _STDIN_PLAYER_ARGV: @@ -1658,7 +1671,7 @@ def _speak_v1( # in every output mode — a returned BaseResult is only printed and # still exits 0. Reachable now that unknown models (e.g. a bare # "flux" typo) route here instead of the raising v2 path. - raise click.ClickException(f"Error generating speech: {e}") + raise _failed("Error generating speech", e) from e finally: _close(audio_iter) diff --git a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py index bc03c6e9..b513fd84 100644 --- a/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py +++ b/packages/deepctl-cmd-speak/tests/unit/test_speak_command.py @@ -802,6 +802,47 @@ def _boom(): file=None, ) + @patch("deepctl_cmd_speak.command.sys") + def test_flux_failure_shows_the_cause_not_the_sdk_fields( + self, + mock_sys, + command, + mock_config, + mock_auth_manager, + mock_client, + tmp_path, + ): + """The client's ApiError wrapper shows as its cause's own words.""" + from deepgram.core.api_error import ApiError + + mock_sys.stdin.isatty.return_value = True + mock_sys.stdout.isatty.return_value = True + + def _boom(): + try: + raise RuntimeError("socket closed") + except RuntimeError as e: + raise ApiError(body=f"Text-to-speech streaming failed: {e}") + yield # pragma: no cover — make this a generator + + mock_client.speak_text_stream.return_value = _boom() + + with pytest.raises(click.ClickException) as err: + command.handle( + config=mock_config, + auth_manager=mock_auth_manager, + client=mock_client, + text="Hello from Flux", + output=str(tmp_path / "x.wav"), + model="flux-alexis-en", + encoding=None, + container=None, + sample_rate=None, + file=None, + ) + assert err.value.message == "Flux streaming failed: socket closed" + assert "status_code" not in err.value.message + @patch("deepctl_cmd_speak.command.sys") def test_handle_flux_streams_to_stdout_incrementally( self, mock_sys, command, mock_config, mock_auth_manager, mock_client diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py new file mode 100644 index 00000000..3b21004f --- /dev/null +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/channel.py @@ -0,0 +1,188 @@ +"""Which tool installed this copy of deepctl, judged from its own layout. + +Stdlib only: the startup check uses these on every run. + +Each test looks at ``sys.prefix``, the environment deepctl runs in, not at +the interpreter it links to. A pipx or ``uv tool`` environment built on +Homebrew's Python resolves its interpreter into Homebrew's Cellar, so the +interpreter can't tell those installs from Homebrew's own ``deepgram`` +formula. +""" + +from __future__ import annotations + +import os +import re +import sys +from itertools import pairwise +from pathlib import Path + +LAUNCHER_ENV_VAR = "DEEPCTL_LAUNCHER" +NPM_LAUNCHERS = ("npm", "npx") + +_UV_ARCHIVE = re.compile(r"archive-v\d+") + + +def launcher() -> str | None: + """``npm`` or ``npx`` when the @deepgram/cli npm package started dg. + + ``deepctl.main`` removes ``DEEPCTL_LAUNCHER`` from the environment at + startup and keeps its value; the environment is the fallback when dg + didn't start through ``deepctl.main``. + """ + main = sys.modules.get("deepctl.main") + if main is not None and getattr(main, "LAUNCHER_CAPTURED", False): + value = getattr(main, "LAUNCHER", None) + else: + value = os.environ.get(LAUNCHER_ENV_VAR) + value = (value or "").strip().lower() + return value if value in NPM_LAUNCHERS else None + + +def _home() -> Path | None: + try: + return Path.home() + except (KeyError, RuntimeError): + return None + + +def _env_path(name: str) -> Path | None: + value = os.environ.get(name) + return Path(value) if value else None + + +def _prefixes() -> list[Path]: + """``sys.prefix`` as given and with links resolved.""" + prefix = Path(sys.prefix) + paths = [prefix] + try: + resolved = prefix.resolve() + except OSError: + return paths + if resolved != prefix: + paths.append(resolved) + return paths + + +def _parts_below(base: Path) -> tuple[str, ...] | None: + """The parts of ``sys.prefix`` below ``base``, or ``None`` if it isn't + inside ``base`` (each compared as given and with links resolved).""" + bases = [base] + try: + resolved = base.resolve() + except OSError: + resolved = base + if resolved != base: + bases.append(resolved) + for prefix in _prefixes(): + for candidate in bases: + try: + return prefix.relative_to(candidate).parts + except ValueError: + continue + return None + + +def _first(paths: list[Path | None]) -> list[Path]: + """The given paths without ``None`` and duplicates, in order.""" + seen: list[Path] = [] + for path in paths: + if path is not None and path not in seen: + seen.append(path) + return seen + + +def is_brew() -> bool: + """Homebrew's ``deepgram`` formula: ``sys.prefix`` under + ``Cellar/deepgram/``.""" + for prefix in _prefixes(): + parts = prefix.parts + if any(pair == ("Cellar", "deepgram") for pair in pairwise(parts)): + return True + return False + + +def uv_cache_dirs() -> list[Path]: + """Where uv keeps its cache: ``UV_CACHE_DIR``, else its defaults.""" + home = _home() + xdg = _env_path("XDG_CACHE_HOME") + local = _env_path("LOCALAPPDATA") + return _first( + [ + _env_path("UV_CACHE_DIR"), + xdg / "uv" if xdg else None, + home / ".cache" / "uv" if home else None, + local / "uv" / "cache" if local and sys.platform == "win32" else None, + ] + ) + + +def is_uvx() -> bool: + """``uvx deepctl``: an environment in uv's cache (``archive-v*``).""" + for cache in uv_cache_dirs(): + parts = _parts_below(cache) + if parts and _UV_ARCHIVE.fullmatch(parts[0]): + return True + return False + + +def uv_tool_dirs() -> list[Path]: + """Where ``uv tool install`` puts tools: ``UV_TOOL_DIR``, else its + defaults.""" + home = _home() + xdg = _env_path("XDG_DATA_HOME") + appdata = _env_path("APPDATA") + return _first( + [ + _env_path("UV_TOOL_DIR"), + xdg / "uv" / "tools" if xdg else None, + home / ".local" / "share" / "uv" / "tools" if home else None, + appdata / "uv" / "data" / "tools" + if appdata and sys.platform == "win32" + else None, + ] + ) + + +def is_uv_tool() -> bool: + """``uv tool install deepctl``: ``sys.prefix`` is ``/``.""" + return any(len(_parts_below(d) or ()) == 1 for d in uv_tool_dirs()) + + +def pipx_homes() -> list[Path]: + """pipx's home: ``PIPX_HOME``, else the locations pipx versions use.""" + home = _home() + xdg = _env_path("XDG_DATA_HOME") + return _first( + [ + _env_path("PIPX_HOME"), + xdg / "pipx" if xdg else None, + home / ".local" / "share" / "pipx" if home else None, + home / ".local" / "pipx" if home else None, + home / "Library" / "Application Support" / "pipx" if home else None, + home / "pipx" if home else None, + ] + ) + + +def _in_pipx(area: str) -> bool: + return any( + len(parts) == 2 and parts[0] == area + for parts in (_parts_below(home) for home in pipx_homes()) + if parts + ) + + +def is_pipx() -> bool: + """``pipx install deepctl``: ``sys.prefix`` is ``/venvs/``.""" + return _in_pipx("venvs") + + +def is_pipx_run() -> bool: + """``pipx run deepctl``: ``sys.prefix`` is ``/.cache/``.""" + return _in_pipx(".cache") + + +def is_oneshot() -> bool: + """A run that fetches deepctl for one use: uvx, pipx run or npx.""" + return is_uvx() or is_pipx_run() or launcher() == "npx" diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py new file mode 100644 index 00000000..ecf7ef8a --- /dev/null +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/check_cache.py @@ -0,0 +1,159 @@ +"""When the update checks last ran, kept in dg's cache directory. + +Each check has one small JSON file there, ``update-check.json`` for dg itself +and ``plugin-update-check.json`` for plugins, holding ``checked_at`` (Unix +time), ``ok``, ``latest`` and ``source``, plus ``notified_at`` once a notice +was shown. A check writes its file just before it sends its request and again +when it gets an answer, so a check that fails or never finishes still waits a +full interval before the next attempt, while a run that ends before the +request goes out doesn't use up the interval. + +The user config file is never written here. The directory is 0700 and must be +a real directory owned by this user: :func:`usable` tightens one this user +owns that its group or others can reach (``mkdir`` under umask 002 leaves +0775), and reports anything else as unusable, in which case the checks don't +run at all. Files are 0600 and replaced atomically (``deepctl._cache``). A +file that isn't a regular 0600-style file owned by this user, or any error, +reads as "no state". +""" + +from __future__ import annotations + +import json +import os +import stat +import sys +import time +from typing import Any + +UPDATE_FILE = "update-check.json" +PLUGIN_FILE = "plugin-update-check.json" + + +def _root() -> str | None: + try: + from deepctl import _cache + except ImportError: + return None + return _cache.cache_root() + + +def usable() -> bool: + """Whether the checks may run: the cache directory exists (created 0700 + if missing), is a real directory owned by this user, and is private + (tightened to 0700 here if its group or others had access). + + ``False`` means: skip the checks this run, with no network and no state. + """ + root = _root() + if root is None: + return False + try: + os.makedirs(root, mode=0o700, exist_ok=True) + except OSError: + return False + if sys.platform == "win32": + return os.path.isdir(root) and not os.path.islink(root) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(root, flags) + except OSError: + return False + try: + info = os.fstat(fd) + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid(): + return False + if stat.S_IMODE(info.st_mode) & 0o077: + os.fchmod(fd, 0o700) + return not stat.S_IMODE(os.fstat(fd).st_mode) & 0o077 + return True + except OSError: + return False + finally: + os.close(fd) + + +def _directory() -> str | None: + """The cache directory, or ``None`` if it's missing, unsafe or unknown.""" + root = _root() + if root is None: + return None + from deepctl import _cache + + return root if _cache._safe_dir(root) else None + + +def read(name: str) -> dict[str, Any] | None: + """The state saved in ``name``, or ``None``.""" + directory = _directory() + if directory is None: + return None + from deepctl import _cache + + data = _cache._read_private(os.path.join(directory, name)) + if data is None: + return None + try: + state = json.loads(data) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def _save(name: str, state: dict[str, Any]) -> None: + directory = _directory() + if directory is None: + return + from deepctl import _cache + + try: + _cache._write_private(os.path.join(directory, name), json.dumps(state).encode()) + except OSError: + pass + + +def write(name: str, *, ok: bool, latest: str | None, source: str) -> None: + """Save the state of a check that ran now. Never raises.""" + _save( + name, + {"checked_at": time.time(), "ok": ok, "latest": latest, "source": source}, + ) + + +def mark_notified(name: str) -> None: + """Record that the notice for the saved state was shown now.""" + state = read(name) + if state is not None: + state["notified_at"] = time.time() + _save(name, state) + + +def _time(state: dict[str, Any] | None, key: str) -> float: + value = (state or {}).get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0 + return float(value) + + +def checked_at(state: dict[str, Any] | None) -> float: + """When the check in ``state`` ran (``0.0`` if unknown).""" + return _time(state, "checked_at") + + +def notified_at(state: dict[str, Any] | None) -> float: + """When the notice for ``state`` was last shown (``0.0`` if never).""" + return _time(state, "notified_at") + + +def elapsed(since: float, interval: float) -> bool: + """Whether at least ``interval`` seconds passed since ``since``. + + A time in the future (a clock that moved back) counts as passed. + """ + gone = time.time() - since + return not 0 <= gone < interval + + +def due(name: str, interval: float) -> bool: + """Whether at least ``interval`` seconds passed since the last check.""" + return elapsed(checked_at(read(name)), interval) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py index 10743c6d..b66df1da 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/command.py @@ -14,12 +14,17 @@ print_success, print_warning, ) +from rich.markup import escape from rich.panel import Panel from rich.prompt import Confirm from .installation import InstallationDetector from .models import UpdateResult -from .version_check import VersionChecker, format_version_message +from .version_check import ( + UpdateCheckError, + VersionChecker, + format_version_message, +) # asyncio (and with it ssl) loads only when an update check runs, so @@ -42,6 +47,7 @@ class UpdateCommand(BaseCommand): examples = [ "dg update", "dg update --check-only", + "dg update --dry-run", ] agent_help = ( "Check for and install updates to deepctl. Can check PyPI for newer " @@ -58,6 +64,15 @@ def get_arguments(self) -> list[dict[str, Any]]: "is_flag": True, "is_option": True, }, + { + "names": ["--dry-run"], + "help": ( + "Show how deepctl was installed and the command that " + "updates it, without checking for or installing updates" + ), + "is_flag": True, + "is_option": True, + }, { "names": ["--force"], "help": "Force update even if already up to date", @@ -94,6 +109,8 @@ def handle( # Extract arguments from kwargs check_only = kwargs.get("check_only", False) + if kwargs.get("dry_run", False): + return self._dry_run() force = kwargs.get("force", False) yes = kwargs.get("yes", False) @@ -107,9 +124,12 @@ def handle( version_checker = VersionChecker(config, current_version) # Check for updates + unreachable: UpdateCheckError | None = None with console.status("Checking for updates..."): try: version_info = asyncio.run(version_checker.check_version(force=True)) + except UpdateCheckError as e: + unreachable = e except Exception as e: print_error(f"Failed to check for updates: {e}") # status="error" on the model, not a dict: see the docstring. @@ -119,6 +139,15 @@ def handle( success=False, message=f"Failed to check for updates: {e}", ) + if unreachable is not None: + # Never "you're on the latest version": that's unknown. A coded + # error (update_check_failed, exit 1) with ERROR/HINT on stderr. + from deepctl_core.transport import TransportError + + host = unreachable.host + raise TransportError( + "update_check_failed", values={"host": host}, details={"host": host} + ) # Display version info message = format_version_message(version_info) @@ -153,15 +182,8 @@ def handle( detector = InstallationDetector() install_info = detector.detect() - # Store installation info for future use — serialize enum to its string value - config._set_config_value( - "update.installation_method", install_info.method.value - ) - config._set_config_value("update.installation_path", install_info.path) - config.save(implicit=True) - # Display installation info - console.print(f"Installation method: [cyan]{install_info.method}[/cyan]") + console.print(f"Installation method: [cyan]{install_info.method.value}[/cyan]") console.print(f"Installation path: [dim]{install_info.path}[/dim]") if install_info.virtual_env: console.print("[yellow]Virtual environment detected[/yellow]") @@ -219,10 +241,6 @@ def handle( f"Successfully updated to version {version_info.latest_version}" ) - # Clear version cache - config._set_config_value("update.cached_version_info", None) - config.save(implicit=True) - return UpdateResult( success=True, message=f"Successfully updated to version {version_info.latest_version}", @@ -265,3 +283,31 @@ def handle( update_available=version_info.update_available, installation_method=install_info.method.value, ) + + def _dry_run(self) -> dict[str, Any]: + """Print the installation method and its update command; change + nothing and make no network request. + + The lines print only in the default format: in the others, stdout + carries just the payload, whose ``message`` holds the same command. + """ + from deepctl_core.output import get_output_format + + detector = InstallationDetector() + install_info = detector.detect() + update_command = detector.get_update_command(install_info.method) + if update_command: + message = f"Run: {shlex.join(update_command)}" + else: + message = detector.get_update_instructions(install_info) + if get_output_format() == "default": + console = get_console() + method = install_info.method.value + console.print(f"Installation method: [cyan]{method}[/cyan]") + console.print(f"Installation path: [dim]{escape(install_info.path)}[/dim]") + console.print(message, markup=False, highlight=False) + return UpdateResult( + success=True, + message=message, + installation_method=install_info.method.value, + ).model_dump() diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py index 6fda2fe9..c11345d6 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py @@ -9,6 +9,10 @@ from pydantic import BaseModel +from . import channel + +NPM_UPDATE_COMMAND = "npm i -g @deepgram/cli@latest" + class InstallMethod(str, Enum): """Supported installation methods.""" @@ -20,6 +24,7 @@ class InstallMethod(str, Enum): UV_TOOL = "uv_tool" UVX = "uvx" PIPX_RUN = "pipx_run" + NPM = "npm" SYSTEM = "system" DEVELOPMENT = "development" UNKNOWN = "unknown" @@ -97,14 +102,16 @@ def get_update_command(self, method: InstallMethod) -> list[str] | None: "deepctl", ], InstallMethod.HOMEBREW: ["brew", "upgrade", "deepgram"], + # `uv tool upgrade` leaves a pinned install where it is. InstallMethod.UV_TOOL: [ "uv", "tool", - "upgrade", - "deepctl", + "install", + "deepctl@latest", ], InstallMethod.UVX: None, # Ephemeral, no update needed InstallMethod.PIPX_RUN: None, # Ephemeral, no update needed + InstallMethod.NPM: None, # Printed, not run: npm owns the install InstallMethod.SYSTEM: None, # Handled separately InstallMethod.DEVELOPMENT: None, # Can't auto-update InstallMethod.UNKNOWN: None, @@ -125,10 +132,22 @@ def get_update_instructions(self, info: InstallationInfo) -> str: if info.method == InstallMethod.UVX: return ( - "You are running deepctl via uvx (one-shot execution). " - "The latest version will be used automatically on next run." + "uvx runs the latest deepctl each time, so there's nothing to " + "update. If you also installed deepctl with uv tool install, " + "uvx runs that copy instead; update it with: " + "uv tool install deepctl@latest. " + "To pin a version, run uvx deepctl@." ) + if info.method == InstallMethod.NPM: + if channel.launcher() == "npx": + return ( + "npx runs the latest @deepgram/cli each time, so there's " + "nothing to update. To pin a version, run " + "npx @deepgram/cli@." + ) + return f"Run: {NPM_UPDATE_COMMAND}" + if info.method == InstallMethod.PIPX_RUN: return ( "You are running deepctl via pipx run (one-shot execution). " @@ -263,8 +282,8 @@ def _detect_method( ) -> InstallMethod: """Detect the installation method. - Priority: development > oneshot > homebrew > uv_tool > pipx > - uv > system > pip > unknown + Priority: development > npm > oneshot > homebrew > uv_tool > + pipx > uv > system > pip > unknown Args: package_location: Path to package @@ -278,6 +297,10 @@ def _detect_method( if is_editable: return InstallMethod.DEVELOPMENT + # Started by the @deepgram/cli npm package (npm or npx) + if channel.launcher() is not None: + return InstallMethod.NPM + # One-shot execution (uvx / pipx run) oneshot = self._detect_oneshot_execution() if oneshot is not None: @@ -313,24 +336,16 @@ def _detect_method( return InstallMethod.UNKNOWN def _is_homebrew_install(self) -> bool: - """Check if installed via Homebrew. + """Check if installed by Homebrew's ``deepgram`` formula. - Detects Apple Silicon (/opt/homebrew), Intel Mac - (/usr/local/Cellar), and Linux Homebrew paths. + Only ``sys.prefix`` under ``Cellar/deepgram/`` counts: a pipx or + uv tool environment on Homebrew's Python resolves its interpreter + into the Cellar too, and isn't a Homebrew install of deepctl. Returns: True if Homebrew installation """ - exe_path = Path(sys.executable).resolve() - exe_str = str(exe_path) - - homebrew_prefixes = [ - "/opt/homebrew", # Apple Silicon - "/usr/local/Cellar", # Intel Mac - "/home/linuxbrew/.linuxbrew", # Linux - ] - - return any(exe_str.startswith(prefix) for prefix in homebrew_prefixes) + return channel.is_brew() def _is_uv_tool_install(self) -> bool: """Check if installed via ``uv tool install``. @@ -338,24 +353,7 @@ def _is_uv_tool_install(self) -> bool: Returns: True if uv tool installation """ - # Use sys.prefix (the venv root), not sys.executable — uv's Python - # interpreter is a symlink to a shared binary pool outside tools/, - # so resolve() on the executable breaks the path check. - prefix_str = str(Path(sys.prefix)) - exe_str = str(Path(sys.executable)) - - # Check custom UV_TOOL_DIR env var first - uv_tool_dir = os.environ.get("UV_TOOL_DIR") - if uv_tool_dir and ( - prefix_str.startswith(uv_tool_dir) or exe_str.startswith(uv_tool_dir) - ): - return True - - # Default uv tool location - default_uv_tools = str(Path.home() / ".local" / "share" / "uv" / "tools") - return prefix_str.startswith(default_uv_tools) or exe_str.startswith( - default_uv_tools - ) + return channel.is_uv_tool() def _detect_oneshot_execution(self) -> InstallMethod | None: """Detect ephemeral one-shot execution (uvx / pipx run). @@ -363,17 +361,10 @@ def _detect_oneshot_execution(self) -> InstallMethod | None: Returns: UVX or PIPX_RUN if detected, None otherwise """ - # uvx sets UV_INTERNAL__PARENT_INTERPRETER - if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: + if channel.is_uvx(): return InstallMethod.UVX - - # pipx run uses a cache directory - exe_path = str(Path(sys.executable).resolve()) - pipx_home = os.environ.get("PIPX_HOME", str(Path.home() / ".local" / "pipx")) - pipx_cache = str(Path(pipx_home) / ".cache") - if exe_path.startswith(pipx_cache): + if channel.is_pipx_run(): return InstallMethod.PIPX_RUN - return None def _is_pipx_install(self) -> bool: @@ -382,13 +373,7 @@ def _is_pipx_install(self) -> bool: Returns: True if pipx installation """ - # Check for pipx environment variables - if "PIPX_HOME" in os.environ or "PIPX_BIN_DIR" in os.environ: - return True - - # Check if executable is in a pipx-managed location - exe_path = Path(sys.executable) - return "pipx" in str(exe_path) or (exe_path.parent.parent / ".pipx").exists() + return channel.is_pipx() def _is_uv_install(self) -> bool: """Check if installed via uv. diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py index e490cb3b..d4783964 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py @@ -2,7 +2,10 @@ Discovers community plugins (excluding first-party packages) and checks PyPI for newer versions. Runs in a daemon thread alongside the core -update check and prints notifications to stderr after CLI execution. +update check, under the same rules: skipped when the core check is, +at most once per interval with its state in +``/plugin-update-check.json``, and its notice prints after CLI +execution only if the thread already finished (dg never waits for it). """ from __future__ import annotations @@ -10,19 +13,17 @@ import importlib.metadata import sys import threading -import time -from pathlib import Path from typing import Any -from .startup_check import _get_check_interval_seconds, _is_ci, _is_oneshot +from . import check_cache +from .startup_check import _get_check_interval_seconds, suppressed + +THREAD_NAME = "deepctl-plugin-update-check" # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- -_CACHE_DIR = Path.home() / ".cache" / "deepctl" -_PLUGIN_CACHE_FILE = _CACHE_DIR / "last_plugin_version_check" - # First-party packages are version-locked to deepctl — never check these. _EXCLUDED_PREFIXES = ("deepctl-cmd-", "deepctl-core", "deepctl-shared-") _EXCLUDED_EXACT = frozenset({"deepctl", "deepctl-plugin-example"}) @@ -42,27 +43,6 @@ def _is_excluded(name: str) -> bool: return any(name.startswith(prefix) for prefix in _EXCLUDED_PREFIXES) -def _read_plugin_cache_timestamp() -> float: - """Read the last plugin-check timestamp from the cache file. - - Returns: - Unix timestamp of the last check, or ``0.0`` if not available. - """ - try: - return float(_PLUGIN_CACHE_FILE.read_text().strip()) - except (FileNotFoundError, ValueError, OSError): - return 0.0 - - -def _write_plugin_cache_timestamp() -> None: - """Persist the current time to the plugin cache file.""" - try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) - _PLUGIN_CACHE_FILE.write_text(str(time.time())) - except OSError: - pass - - # --------------------------------------------------------------------------- # Discovery # --------------------------------------------------------------------------- @@ -131,9 +111,13 @@ def _discover_community_plugins() -> dict[str, str]: def _check_pypi_versions( plugins: dict[str, str], + record: bool = False, ) -> list[dict[str, str]]: """Query PyPI for each plugin and return those with available updates. + With ``record``, the attempt is saved once the client is built, just + before the first request goes out. + Returns: List of ``{"name": ..., "current": ..., "latest": ...}`` dicts for plugins where a newer version exists on PyPI. @@ -148,6 +132,10 @@ def _check_pypi_versions( from packaging import version as pkg_version with httpx.Client(timeout=5.0) as client: + if record: + check_cache.write( + check_cache.PLUGIN_FILE, ok=False, latest=None, source="pypi" + ) for name, current_ver in list(plugins.items())[:_MAX_PLUGINS_TO_CHECK]: try: resp = client.get(f"https://pypi.org/pypi/{name}/json") @@ -177,19 +165,21 @@ def _check_pypi_versions( def _background_plugin_check(result: dict[str, Any]) -> None: - """Run plugin discovery + PyPI check (called inside a daemon thread).""" + """Run plugin discovery + PyPI check (called inside a daemon thread). + + The attempt is saved just before the first request goes out (see + :func:`_check_pypi_versions`), so one that fails or never finishes still + counts as this interval's check, while a run that ends before then + doesn't. + """ try: plugins = _discover_community_plugins() - if not plugins: - return - - updates = _check_pypi_versions(plugins) - if updates: - result["updates"] = updates - - _write_plugin_cache_timestamp() + updates = _check_pypi_versions(plugins, record=True) if plugins else [] except Exception: - pass + return + check_cache.write(check_cache.PLUGIN_FILE, ok=True, latest=None, source="pypi") + if updates: + result["updates"] = updates # --------------------------------------------------------------------------- @@ -216,22 +206,27 @@ def check_plugins_and_notify(quiet: bool = False) -> None: quiet: If True, suppress the check entirely. """ global _thread, _result + _thread = None _result = {} - if quiet or _is_ci() or _is_oneshot(): + if suppressed(quiet): return interval = _get_check_interval_seconds(quiet) if interval is None: return - last_check = _read_plugin_cache_timestamp() - if time.time() - last_check < interval: + # A cache directory that can't be made private: no check, no state. + if not check_cache.usable(): + return + + if not check_cache.due(check_cache.PLUGIN_FILE, interval): return _thread = threading.Thread( target=_background_plugin_check, args=(_result,), + name=THREAD_NAME, daemon=True, ) _thread.start() @@ -240,18 +235,16 @@ def check_plugins_and_notify(quiet: bool = False) -> None: def print_pending_plugin_notifications() -> None: """Print plugin update notifications if any are pending. - Call this **after** CLI execution completes. It joins the background - thread (with a short timeout) and, if updates were found, prints - ANSI-colored lines to stderr. + Call this **after** CLI execution completes. It never waits: if the + thread already finished and found updates, it prints ANSI-colored + lines to stderr; a check that is still running is dropped. """ global _thread - if _thread is None: + thread, _thread = _thread, None + if thread is None or thread.is_alive(): return - _thread.join(timeout=3.0) - _thread = None - updates = _result.get("updates") if not updates: return diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py index 889bab17..fbd8d148 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py @@ -1,8 +1,23 @@ """Background startup update check for deepctl. -Runs a non-intrusive version check in a daemon thread during normal CLI -execution, then prints a one-line notification to stderr *after* the -command output completes. +Runs a version check in a daemon thread while the command runs. When the +command is done, a one-line notice goes to stderr only if the thread has +already finished; dg never waits for it. A check that hasn't finished is +dropped, and the interpreter stops the thread at exit. + +The check runs at most once per interval (daily by default). Its state lives +in ``/update-check.json`` (``check_cache``), written just before the +request goes out and again with the answer, so a failing or slow index is +tried again only after a full interval. When no check is due and the saved +answer names a newer version, the notice prints from it, at most once per +interval (``notified_at``), so an answer that came in too late for its own +run still reaches the person. The user config file is never written. When +the cache directory can't be made private, the check doesn't run at all. + +The check is skipped entirely under ``--quiet``, in CI, in an AI agent or +other non-interactive session, under ``dg mcp``, when stderr isn't a +terminal, for one-shot runs (uvx, pipx run, npx) and when +``DEEPCTL_NO_UPDATE_CHECK`` is set. """ from __future__ import annotations @@ -11,12 +26,13 @@ import os import sys import threading -import time -from pathlib import Path from typing import Any -# Sentinel used when the background check is suppressed or has no result -_NO_RESULT: dict[str, Any] = {} +from . import channel, check_cache + +PYPI_URL = "https://pypi.org/pypi/deepctl/json" +NO_UPDATE_CHECK_ENV_VAR = "DEEPCTL_NO_UPDATE_CHECK" +THREAD_NAME = "deepctl-update-check" # Environment variables that indicate a CI environment _CI_ENV_VARS = ( @@ -31,11 +47,6 @@ "CODEBUILD_BUILD_ID", ) -# Cache file for last-check timestamp (separate from YAML config to -# avoid race conditions) -_CACHE_DIR = Path.home() / ".cache" / "deepctl" -_CACHE_FILE = _CACHE_DIR / "last_version_check" - def _is_ci() -> bool: """Return True if running inside a CI environment.""" @@ -43,36 +54,63 @@ def _is_ci() -> bool: def _is_oneshot() -> bool: - """Return True if running in an ephemeral one-shot environment.""" - # uvx - if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: - return True - # pipx run - exe_path = str(Path(sys.executable).resolve()) - pipx_home = os.environ.get("PIPX_HOME", str(Path.home() / ".local" / "pipx")) - pipx_cache = str(Path(pipx_home) / ".cache") - return bool(exe_path.startswith(pipx_cache)) - - -def _read_cache_timestamp() -> float: - """Read the last-check timestamp from the cache file. - - Returns: - Unix timestamp of the last check, or ``0.0`` if not available. - """ + """Return True for a one-shot run (uvx, pipx run, npx): nothing to update.""" + return channel.is_oneshot() + + +def _opted_out() -> bool: + """``DEEPCTL_NO_UPDATE_CHECK`` is set to anything but empty, 0 or false.""" + value = os.environ.get(NO_UPDATE_CHECK_ENV_VAR, "").strip().lower() + return value not in ("", "0", "false", "no") + + +def _stderr_is_terminal() -> bool: try: - return float(_CACHE_FILE.read_text().strip()) - except (FileNotFoundError, ValueError, OSError): - return 0.0 + return bool(sys.stderr is not None and sys.stderr.isatty()) + except (AttributeError, OSError, ValueError): + return False -def _write_cache_timestamp() -> None: - """Persist the current time to the cache file.""" +def _is_agent() -> bool: + """An AI agent or another non-interactive session (deepctl_core's rules).""" + try: + from deepctl_core.output import is_agentic, is_ai_agent + except ImportError: + return False + return is_agentic() or is_ai_agent() + + +def _subcommand() -> str | None: + """The command this run executes (``mcp`` for ``dg -q mcp``), if known.""" + cli = sys.modules.get("deepctl._cli") + args = getattr(cli, "_run_args", None) + if args is None: + args = sys.argv[1:] try: - _CACHE_DIR.mkdir(parents=True, exist_ok=True) - _CACHE_FILE.write_text(str(time.time())) - except OSError: - pass + flag_opts = cli._root_flag_opts() if cli is not None else set() + i = 0 + while i < len(args) and args[i].startswith("-"): + takes_value = cli is not None and cli._takes_next_token(args[i], flag_opts) + i += 2 if takes_value else 1 + except Exception: + return None + return args[i] if i < len(args) else None + + +def suppressed(quiet: bool) -> bool: + """Whether this run must not check for updates at all. + + Shared by the plugin-update check. Cheap: no config or network access. + """ + return ( + quiet + or _opted_out() + or _is_ci() + or not _stderr_is_terminal() + or _is_agent() + or _is_oneshot() + or _subcommand() == "mcp" + ) def _get_check_interval_seconds(quiet: bool) -> float | None: @@ -101,43 +139,78 @@ def _get_check_interval_seconds(quiet: bool) -> float | None: def _background_check(current_version: str, result: dict[str, Any]) -> None: - """Run the version check synchronously (called inside a thread).""" + """Run the version check synchronously (called inside a thread). + + The attempt is saved just before the request goes out, so one that + fails or never finishes still counts as this interval's check, while a + run that ends before the request (imports take a while) doesn't. + """ try: + # httpx loads here, in the thread, never on the main path. Building + # the client (CA bundle, proxy settings) also comes before the record. import httpx - - response = httpx.get( - "https://pypi.org/pypi/deepctl/json", - timeout=5.0, - ) - response.raise_for_status() - data = response.json() - latest_version: str = data["info"]["version"] - from packaging import version as pkg_version - current = pkg_version.parse(current_version) - latest = pkg_version.parse(latest_version) + client = httpx.Client(timeout=5.0) + except Exception: + return + with client: + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + try: + response = client.get(PYPI_URL) + response.raise_for_status() + latest_version = response.json()["info"]["version"] + if not isinstance(latest_version, str): + return + newer = pkg_version.parse(latest_version) > pkg_version.parse( + current_version + ) + except Exception: + # Best effort: the failure is already recorded above. + return - if latest > current: - result["latest"] = latest_version - result["current"] = current_version + check_cache.write( + check_cache.UPDATE_FILE, ok=True, latest=latest_version, source="pypi" + ) + if newer: + result["latest"] = latest_version + result["current"] = current_version + + +def _newer_saved( + state: dict[str, Any] | None, current_version: str, interval: float +) -> tuple[str, str] | None: + """``(current, latest)`` when the saved answer names a newer version + whose notice wasn't shown within ``interval``.""" + if not state or state.get("ok") is not True: + return None + latest = state.get("latest") + if not isinstance(latest, str): + return None + if not check_cache.elapsed(check_cache.notified_at(state), interval): + return None + try: + from packaging import version as pkg_version - _write_cache_timestamp() + if pkg_version.parse(latest) <= pkg_version.parse(current_version): + return None except Exception: - # Silently swallow — this is best-effort - pass + return None + return current_version, latest # Module-level state for the background thread _thread: threading.Thread | None = None _result: dict[str, Any] = {} +# A notice from the saved answer, when no check ran this time. +_saved_notice: tuple[str, str] | None = None def check_and_notify( current_version: str | None = None, quiet: bool = False, ) -> None: - """Start a background version check. + """Start a background version check, unless this run skips it. Call this **before** CLI execution. It spawns a daemon thread that runs concurrently. Call :func:`print_pending_notification` after CLI @@ -148,20 +221,20 @@ def check_and_notify( detected automatically from package metadata. quiet: If True, suppress the check entirely. """ - global _thread, _result + global _thread, _result, _saved_notice + _thread = None _result = {} + _saved_notice = None - # Suppress in quiet mode, CI, or one-shot execution - if quiet or _is_ci() or _is_oneshot(): + if suppressed(quiet): return - # Check interval interval = _get_check_interval_seconds(quiet) if interval is None: return - last_check = _read_cache_timestamp() - if time.time() - last_check < interval: + # A cache directory that can't be made private: no check, no state. + if not check_cache.usable(): return # Resolve current version @@ -171,39 +244,50 @@ def check_and_notify( except importlib.metadata.PackageNotFoundError: return + state = check_cache.read(check_cache.UPDATE_FILE) + if not check_cache.elapsed(check_cache.checked_at(state), interval): + _saved_notice = _newer_saved(state, current_version, interval) + return + _thread = threading.Thread( target=_background_check, args=(current_version, _result), + name=THREAD_NAME, daemon=True, ) _thread.start() def print_pending_notification() -> None: - """Print the update notification if one is pending. + """Print the update notice if the check already found one. - Call this **after** CLI execution completes. It joins the background - thread (with a short timeout) and, if an update was found, prints a - single ANSI-colored line to stderr. + Call this **after** CLI execution completes. It never waits: a check + that is still running is dropped (its answer, once saved, gives the + notice on a later run). Without a check this run, a saved answer that + names a newer version prints instead, at most once per interval. """ - global _thread + global _thread, _saved_notice - if _thread is None: + thread, _thread = _thread, None + notice, _saved_notice = _saved_notice, None + if thread is not None: + if thread.is_alive(): + return + latest = _result.get("latest") + current = _result.get("current") + notice = (current, latest) if latest and current else None + if notice is None: return - _thread.join(timeout=2.0) - _thread = None - - latest = _result.get("latest") - current = _result.get("current") - if latest and current: - try: - sys.stderr.write( - f"\n\033[33m Update available: {current} → {latest}" - f" — run \033[1mdg update\033[0m\033[33m to upgrade\033[0m\n\n" - ) - except (BrokenPipeError, OSError, ValueError): - # The output stream closed (e.g. an MCP host disconnected stdio - # after `dg mcp` finished). Nothing useful to do — the notification - # is purely advisory. - pass + current, latest = notice + try: + sys.stderr.write( + f"\n\033[33m Update available: {current} → {latest}" + f" — run \033[1mdg update\033[0m\033[33m to upgrade\033[0m\n\n" + ) + except (BrokenPipeError, OSError, ValueError): + # The output stream closed (e.g. an MCP host disconnected stdio + # after `dg mcp` finished). Nothing useful to do — the notification + # is purely advisory. + return + check_cache.mark_notified(check_cache.UPDATE_FILE) diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py index ce3e5df5..d46b9f43 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py @@ -1,11 +1,31 @@ -"""Version checking functionality for deepctl.""" +"""Version checking functionality for deepctl. + +The last check's result lives in ``/update-check.json`` (see +``check_cache``), shared with the startup check. Versions before 0.5.0 kept +it in the user config (``update.last_check`` and +``update.cached_version_info``); those keys are still read while no cache +file exists, and never written. +""" from datetime import datetime, timedelta, timezone +from typing import Any from deepctl_core import Config from packaging import version from pydantic import BaseModel, Field +from . import check_cache + +PYPI_HOST = "pypi.org" + + +class UpdateCheckError(Exception): + """No version source answered; ``host`` is the last one tried.""" + + def __init__(self, host: str) -> None: + self.host = host + super().__init__(f"couldn't reach {host}") + class VersionInfo(BaseModel): """Version information from PyPI.""" @@ -44,6 +64,8 @@ def __init__(self, config: Config, current_version: str = "0.0.0"): """ self.config = config self.current_version = current_version + # Where the last successful answer came from ("pypi" or "github"). + self._source = "pypi" def _get_cache_duration(self) -> timedelta | None: """Get the cache duration based on ``check_frequency`` config. @@ -71,6 +93,10 @@ async def check_version( Returns: Version information including update availability + + Raises: + UpdateCheckError: no source answered (PyPI is the last one + tried). Never reported as "up to date". """ # Check if we should skip based on cache if not force and not self.should_check(): @@ -80,6 +106,7 @@ async def check_version( return cached_info # Choose version source + self._source = "pypi" if install_method == "homebrew": version_info = await self._check_github_releases() else: @@ -89,12 +116,11 @@ async def check_version( self._cache_info(version_info) return version_info - # On error, return current version with no update - return VersionInfo( - current_version=self.current_version, - latest_version=self.current_version, - update_available=False, - ) + if check_cache.usable(): + check_cache.write( + check_cache.UPDATE_FILE, ok=False, latest=None, source=self._source + ) + raise UpdateCheckError(PYPI_HOST) async def _check_pypi(self) -> VersionInfo | None: """Fetch latest version from PyPI. @@ -174,6 +200,7 @@ async def _check_github_releases(self) -> VersionInfo | None: ) release_notes_url = data.get("html_url") + self._source = "github" current = version.parse(self.current_version) latest = version.parse(latest_version) @@ -188,6 +215,7 @@ async def _check_github_releases(self) -> VersionInfo | None: ) except Exception: # Fall back to PyPI on GitHub API failure + self._source = "pypi" return await self._check_pypi() def should_check(self) -> bool: @@ -206,7 +234,14 @@ def should_check(self) -> bool: # "never" — always skip return False - # Check last check timestamp + state = self._saved_state() + if state is not None: + elapsed = datetime.now(timezone.utc).timestamp() - check_cache.checked_at( + state + ) + return not 0 <= elapsed < cache_duration.total_seconds() + + # No cache file yet: the timestamp older versions kept in the config last_check = self.config.get("update.last_check") if last_check: try: @@ -222,12 +257,32 @@ def should_check(self) -> bool: return True + @staticmethod + def _saved_state() -> dict[str, Any] | None: + return check_cache.read(check_cache.UPDATE_FILE) + def _get_cached_info(self) -> VersionInfo | None: - """Get cached version info from config. + """Get the cached version info. Returns: Cached version info or None """ + state = self._saved_state() + if state is not None: + latest = state.get("latest") + if not state.get("ok") or not isinstance(latest, str): + return None + try: + available = version.parse(latest) > version.parse(self.current_version) + except version.InvalidVersion: + return None + return VersionInfo( + current_version=self.current_version, + latest_version=latest, + update_available=available, + ) + + # No cache file yet: what older versions kept in the config cached_data = self.config.get("update.cached_version_info") if cached_data: try: @@ -237,18 +292,19 @@ def _get_cached_info(self) -> VersionInfo | None: return None def _cache_info(self, info: VersionInfo) -> None: - """Cache version info in config. + """Save the answer to the cache file (never to the user config). Args: info: Version info to cache """ - self.config._set_config_value( - "update.last_check", datetime.now(timezone.utc).isoformat() - ) - self.config._set_config_value( - "update.cached_version_info", info.model_dump(mode="json") + if not check_cache.usable(): + return + check_cache.write( + check_cache.UPDATE_FILE, + ok=True, + latest=info.latest_version, + source=self._source, ) - self.config.save(implicit=True) def format_version_message(info: VersionInfo) -> str: diff --git a/packages/deepctl-cmd-update/tests/unit/test_installation.py b/packages/deepctl-cmd-update/tests/unit/test_installation.py index 22cd68ff..32141586 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_installation.py +++ b/packages/deepctl-cmd-update/tests/unit/test_installation.py @@ -22,7 +22,7 @@ def test_all_methods_defined(self): """Verify all expected methods exist.""" expected = { "pip", "pipx", "uv", "homebrew", "uv_tool", - "uvx", "pipx_run", "system", "development", "unknown", + "uvx", "pipx_run", "npm", "system", "development", "unknown", } actual = {m.value for m in InstallMethod} assert actual == expected @@ -58,7 +58,7 @@ def test_returns_list_for_homebrew(self, detector): def test_returns_list_for_uv_tool(self, detector): cmd = detector.get_update_command(InstallMethod.UV_TOOL) assert isinstance(cmd, list) - assert cmd == ["uv", "tool", "upgrade", "deepctl"] + assert cmd == ["uv", "tool", "install", "deepctl@latest"] def test_returns_none_for_uvx(self, detector): cmd = detector.get_update_command(InstallMethod.UVX) @@ -77,143 +77,208 @@ def test_returns_none_for_unknown(self, detector): assert cmd is None -class TestHomebrewDetection: - """Test Homebrew installation detection.""" - - @pytest.fixture - def detector(self): +@pytest.fixture +def layout(tmp_path, monkeypatch): + """A home directory with no tool settings, and a way to set sys.prefix.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + for name in ( + "UV_CACHE_DIR", + "UV_TOOL_DIR", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "PIPX_HOME", + "PIPX_BIN_DIR", + "DEEPCTL_LAUNCHER", + "UV_INTERNAL__PARENT_INTERPRETER", + "UV_PROJECT", + ): + monkeypatch.delenv(name, raising=False) + main = sys.modules.get("deepctl.main") + if main is not None: + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", False, raising=False) + + def set_prefix(path): + monkeypatch.setattr(sys, "prefix", str(path)) return InstallationDetector() - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_apple_silicon_path(self, mock_sys): - """Detect Homebrew on Apple Silicon.""" - mock_sys.executable = "/opt/homebrew/bin/python3" - detector = InstallationDetector() - # Patch resolve to return the same path - with patch.object( - Path, "resolve", return_value=Path("/opt/homebrew/bin/python3") - ): - assert detector._is_homebrew_install() is True - - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_intel_mac_path(self, mock_sys): - """Detect Homebrew on Intel Mac.""" - mock_sys.executable = "/usr/local/Cellar/python@3.12/bin/python3" - detector = InstallationDetector() - with patch.object( - Path, - "resolve", - return_value=Path( - "/usr/local/Cellar/python@3.12/bin/python3" - ), - ): - assert detector._is_homebrew_install() is True - - @unix_only - @patch("deepctl_cmd_update.installation.sys") - def test_linux_homebrew_path(self, mock_sys): - """Detect Homebrew on Linux.""" - mock_sys.executable = ( - "/home/linuxbrew/.linuxbrew/bin/python3" + set_prefix.home = home + return set_prefix + + +@unix_only +class TestHomebrewDetection: + """Homebrew means the deepgram formula's own Cellar prefix.""" + + @pytest.mark.parametrize( + "prefix", + [ + "/opt/homebrew/Cellar/deepgram/0.5.0/libexec", + "/usr/local/Cellar/deepgram/0.5.0/libexec", + "/home/linuxbrew/.linuxbrew/Cellar/deepgram/0.5.0/libexec", + ], + ) + def test_formula_prefix(self, layout, prefix): + assert layout(prefix)._is_homebrew_install() is True + + @pytest.mark.parametrize( + "prefix", + [ + "/opt/homebrew/Cellar/python@3.12/3.12.7/Frameworks/Python.framework", + "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/3.12.7", + "/usr", + ], + ) + def test_homebrew_python_alone_is_not_brew(self, layout, prefix): + assert layout(prefix)._is_homebrew_install() is False + + def test_pipx_on_homebrew_python(self, layout, monkeypatch): + """A pipx venv whose interpreter resolves into the Cellar is pipx.""" + prefix = layout.home / ".local" / "share" / "pipx" / "venvs" / "deepctl" + monkeypatch.setattr( + sys, "executable", "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/bin/python3" ) - detector = InstallationDetector() - with patch.object( - Path, - "resolve", - return_value=Path( - "/home/linuxbrew/.linuxbrew/bin/python3" - ), - ): - assert detector._is_homebrew_install() is True + detector = layout(prefix) + assert detector._is_homebrew_install() is False + assert detector._detect_method(prefix, True, False) == InstallMethod.PIPX + + def test_uv_tool_on_homebrew_python(self, layout, monkeypatch): + prefix = layout.home / ".local" / "share" / "uv" / "tools" / "deepctl" + monkeypatch.setattr( + sys, "executable", "/home/linuxbrew/.linuxbrew/Cellar/python@3.12/bin/python3" + ) + detector = layout(prefix) + assert detector._detect_method(prefix, True, False) == InstallMethod.UV_TOOL - @patch("deepctl_cmd_update.installation.sys") - def test_non_homebrew_path(self, mock_sys): - """Non-Homebrew path returns False.""" - mock_sys.executable = "/usr/bin/python3" - detector = InstallationDetector() - with patch.object( - Path, "resolve", return_value=Path("/usr/bin/python3") - ): - assert detector._is_homebrew_install() is False + def test_formula_through_opt_link(self, layout, tmp_path): + cellar = tmp_path / "brew" / "Cellar" / "deepgram" / "0.5.0" / "libexec" + cellar.mkdir(parents=True) + opt = tmp_path / "brew" / "opt" + opt.mkdir() + (opt / "deepgram").symlink_to(cellar.parent) + detector = layout(opt / "deepgram" / "libexec") + assert detector._detect_method(None, True, False) == InstallMethod.HOMEBREW +@unix_only class TestUvToolDetection: """Test uv tool installation detection.""" - @pytest.fixture - def detector(self): - return InstallationDetector() + def test_custom_uv_tool_dir(self, layout, monkeypatch): + monkeypatch.setenv("UV_TOOL_DIR", "/custom/uv/tools") + assert layout("/custom/uv/tools/deepctl")._is_uv_tool_install() is True - @unix_only - @patch("deepctl_cmd_update.installation.sys") - @patch.dict( - "os.environ", {"UV_TOOL_DIR": "/custom/uv/tools"}, clear=False + def test_default_uv_tool_path(self, layout, monkeypatch): + """Detect uv tool at default location via sys.prefix (symlinked exe).""" + monkeypatch.setattr( + sys, "executable", "/x/.local/share/uv/python/cpython-3.11/bin/python3" + ) + prefix = layout.home / ".local" / "share" / "uv" / "tools" / "deepctl" + assert layout(prefix)._is_uv_tool_install() is True + + def test_xdg_data_home(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + prefix = tmp_path / "data" / "uv" / "tools" / "deepctl" + assert layout(prefix)._is_uv_tool_install() is True + + def test_tools_dir_itself_is_not_a_tool(self, layout): + prefix = layout.home / ".local" / "share" / "uv" / "tools" + assert layout(prefix)._is_uv_tool_install() is False + + +@unix_only +class TestPipxDetection: + def test_pipx_home_env(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + assert layout(tmp_path / "px" / "venvs" / "deepctl")._is_pipx_install() + + @pytest.mark.parametrize( + "parts", [(".local", "pipx"), (".local", "share", "pipx")] ) - def test_custom_uv_tool_dir(self, mock_sys): - """Detect uv tool with custom UV_TOOL_DIR via sys.prefix.""" - mock_sys.executable = "/usr/local/share/uv/python/cpython-3.11/bin/python3" - mock_sys.prefix = "/custom/uv/tools/deepctl" - detector = InstallationDetector() - assert detector._is_uv_tool_install() is True + def test_default_homes(self, layout, parts): + prefix = layout.home.joinpath(*parts, "venvs", "deepctl") + assert layout(prefix)._is_pipx_install() is True - @patch("deepctl_cmd_update.installation.sys") - @patch.dict("os.environ", {}, clear=False) - def test_default_uv_tool_path(self, mock_sys): - """Detect uv tool at default location via sys.prefix (symlinked exe).""" - home = str(Path.home()) - # Simulate the real scenario: exe resolves to the shared Python pool - # (outside tools/), but prefix correctly points into tools/ - mock_sys.executable = f"{home}/.local/share/uv/python/cpython-3.11/bin/python3" - mock_sys.prefix = f"{home}/.local/share/uv/tools/deepctl" - detector = InstallationDetector() - assert detector._is_uv_tool_install() is True + def test_pipx_home_set_for_another_install(self, layout, monkeypatch, tmp_path): + """PIPX_HOME in the environment doesn't make a venv a pipx install.""" + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + monkeypatch.setenv("PIPX_BIN_DIR", str(tmp_path / "bin")) + detector = layout(tmp_path / "project" / ".venv") + assert detector._is_pipx_install() is False +@unix_only class TestOneshotDetection: """Test one-shot execution detection (uvx / pipx run).""" - @pytest.fixture - def detector(self): - return InstallationDetector() + def test_uvx_default_cache(self, layout): + prefix = layout.home / ".cache" / "uv" / "archive-v0" / "AbC123" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.UVX - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3"}, - clear=False, - ) - def test_uvx_detected(self, detector): - """uvx is detected via environment variable.""" - result = detector._detect_oneshot_execution() - assert result == InstallMethod.UVX + def test_uvx_with_uv_cache_dir(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uvc")) + prefix = tmp_path / "uvc" / "archive-v0" / "AbC123" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.UVX + + def test_uv_cache_outside_archive_is_not_uvx(self, layout): + prefix = layout.home / ".cache" / "uv" / "builds-v0" / "x" + assert layout(prefix)._detect_oneshot_execution() is None + + def test_parent_interpreter_env_alone_is_not_uvx(self, layout, monkeypatch, tmp_path): + """uv run sets this too, for an ordinary project venv.""" + monkeypatch.setenv("UV_INTERNAL__PARENT_INTERPRETER", "/usr/bin/python3") + assert layout(tmp_path / "project" / ".venv")._detect_oneshot_execution() is None + + def test_pipx_run_detected(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("PIPX_HOME", str(tmp_path / "px")) + prefix = tmp_path / "px" / ".cache" / "0123abcd" + assert layout(prefix)._detect_oneshot_execution() == InstallMethod.PIPX_RUN + + def test_no_oneshot(self, layout, tmp_path): + assert layout(tmp_path / "venv")._detect_oneshot_execution() is None - @unix_only - @patch("deepctl_cmd_update.installation.sys") - @patch.dict("os.environ", {}, clear=False) - def test_pipx_run_detected(self, mock_sys, tmp_path): - """pipx run is detected via cache path.""" - # Create a real file so Path.resolve() works - fake_exe = tmp_path / ".local" / "pipx" / ".cache" / "some" / "bin" / "python3" - fake_exe.parent.mkdir(parents=True) - fake_exe.touch() - mock_sys.executable = str(fake_exe) - with patch.dict( - "os.environ", {"PIPX_HOME": str(tmp_path / ".local" / "pipx")}, - clear=False, - ): - detector = InstallationDetector() - result = detector._detect_oneshot_execution() - assert result == InstallMethod.PIPX_RUN - - @patch.dict("os.environ", {}, clear=False) - def test_no_oneshot(self, detector): - """Regular execution returns None.""" - # Make sure UV_INTERNAL__PARENT_INTERPRETER is not set - import os - os.environ.pop("UV_INTERNAL__PARENT_INTERPRETER", None) - result = detector._detect_oneshot_execution() - assert result is None + +@unix_only +class TestNpmLauncher: + @pytest.mark.parametrize("value", ["npm", "npx"]) + def test_env(self, layout, monkeypatch, tmp_path, value): + monkeypatch.setenv("DEEPCTL_LAUNCHER", value) + detector = layout(tmp_path / "cache" / "venv") + assert detector._detect_method(None, True, False) == InstallMethod.NPM + + def test_captured_by_main(self, layout, monkeypatch, tmp_path): + """deepctl.main removes the variable at startup and keeps its value.""" + import deepctl.main as main + + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", True) + monkeypatch.setattr(main, "LAUNCHER", "npm") + detector = layout(tmp_path / "cache" / "venv") + assert detector._detect_method(None, True, False) == InstallMethod.NPM + + def test_unknown_value_ignored(self, layout, monkeypatch, tmp_path): + monkeypatch.setenv("DEEPCTL_LAUNCHER", "/usr/local/bin/dg") + detector = layout(tmp_path / "venv") + assert detector._detect_method(None, True, False) != InstallMethod.NPM + + def test_npm_prints_the_command(self, layout, monkeypatch, tmp_path): + from deepctl_cmd_update.installation import InstallationInfo + + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npm") + detector = layout(tmp_path / "venv") + assert detector.get_update_command(InstallMethod.NPM) is None + info = InstallationInfo( + method=InstallMethod.NPM, + path="/x", + virtual_env=True, + editable=False, + python_executable="/x/bin/python3", + ) + assert detector.get_update_instructions(info) == ( + "Run: npm i -g @deepgram/cli@latest" + ) + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npx") + assert "npx" in detector.get_update_instructions(info) class TestGetUpdateInstructions: @@ -268,7 +333,7 @@ def test_uvx_instructions(self, detector): ) instructions = detector.get_update_instructions(info) assert "uvx" in instructions - assert "automatically" in instructions + assert "uv tool install deepctl@latest" in instructions def test_pipx_run_instructions(self, detector): from deepctl_cmd_update.installation import InstallationInfo diff --git a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py index 9de3b700..0c5cbc08 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py @@ -3,8 +3,6 @@ import io import sys import threading -import time -from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -13,12 +11,36 @@ _check_pypi_versions, _discover_community_plugins, _is_excluded, - _read_plugin_cache_timestamp, - _write_plugin_cache_timestamp, check_plugins_and_notify, print_pending_plugin_notifications, ) +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture +def check_allowed(monkeypatch, cache_dir): + """A run where the startup checks may start: an interactive person at a + terminal, outside CI and ``dg mcp``. Returns the cache directory.""" + import deepctl_cmd_update.startup_check as core + + monkeypatch.delenv(core.NO_UPDATE_CHECK_ENV_VAR, raising=False) + monkeypatch.setattr(core, "_is_ci", lambda: False) + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: True) + monkeypatch.setattr(core, "_is_agent", lambda: False) + monkeypatch.setattr(core, "_is_oneshot", lambda: False) + monkeypatch.setattr(core, "_subcommand", lambda: None) + monkeypatch.setattr(core, "_get_check_interval_seconds", lambda quiet: 86400.0) + return cache_dir + + # --------------------------------------------------------------------------- # _is_excluded # --------------------------------------------------------------------------- @@ -49,39 +71,6 @@ def test_arbitrary_community_plugin_not_excluded(self): assert _is_excluded("my-deepctl-addon") is False -# --------------------------------------------------------------------------- -# Cache timestamp -# --------------------------------------------------------------------------- - - -class TestPluginCacheTimestamp: - """Test file-based plugin cache timestamp.""" - - def test_read_missing_returns_zero(self, tmp_path): - with patch( - "deepctl_cmd_update.plugin_update_check._PLUGIN_CACHE_FILE", - tmp_path / "nonexistent", - ): - assert _read_plugin_cache_timestamp() == 0.0 - - def test_write_then_read_roundtrip(self, tmp_path): - cache_file = tmp_path / "last_plugin_version_check" - with ( - patch( - "deepctl_cmd_update.plugin_update_check._PLUGIN_CACHE_FILE", - cache_file, - ), - patch( - "deepctl_cmd_update.plugin_update_check._CACHE_DIR", - tmp_path, - ), - ): - _write_plugin_cache_timestamp() - ts = _read_plugin_cache_timestamp() - assert ts > 0.0 - assert abs(ts - time.time()) < 5.0 - - # --------------------------------------------------------------------------- # Discovery # --------------------------------------------------------------------------- @@ -254,43 +243,45 @@ def test_empty_plugins_returns_empty(self): class TestCheckPluginsAndNotify: """Test the plugin check lifecycle.""" - @patch.dict("os.environ", {"CI": "true"}, clear=False) - def test_suppressed_in_ci(self): + @pytest.fixture(autouse=True) + def _reset(self): import deepctl_cmd_update.plugin_update_check as mod mod._thread = None mod._result = {} + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + + def test_suppressed_with_the_core_check(self, check_allowed, monkeypatch): + import deepctl_cmd_update.plugin_update_check as mod + import deepctl_cmd_update.startup_check as core + + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: False) + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() + assert not check_allowed.exists() + + def test_suppressed_in_ci(self, check_allowed, monkeypatch): + import deepctl_cmd_update.plugin_update_check as mod + import deepctl_cmd_update.startup_check as core + + monkeypatch.setattr(core, "_is_ci", lambda: True) check_plugins_and_notify() assert mod._thread is None - def test_suppressed_in_quiet_mode(self): + def test_suppressed_in_quiet_mode(self, check_allowed): import deepctl_cmd_update.plugin_update_check as mod - mod._thread = None - mod._result = {} check_plugins_and_notify(quiet=True) assert mod._thread is None - @patch("deepctl_cmd_update.plugin_update_check._is_ci", return_value=False) - @patch( - "deepctl_cmd_update.plugin_update_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._get_check_interval_seconds", - return_value=86400.0, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._read_plugin_cache_timestamp", - return_value=0.0, - ) - def test_fresh_cache_spawns_thread(self, *_mocks): - """Cache expired → thread should be spawned.""" + def test_due_spawns_thread(self, check_allowed): + """No check within the interval → thread should be spawned.""" import deepctl_cmd_update.plugin_update_check as mod - mod._thread = None - mod._result = {} - with patch( "deepctl_cmd_update.plugin_update_check._background_plugin_check", side_effect=lambda res: None, @@ -298,30 +289,45 @@ def test_fresh_cache_spawns_thread(self, *_mocks): check_plugins_and_notify() assert mod._thread is not None assert mod._thread.daemon is True - mod._thread.join(timeout=2.0) + assert mod._thread.name == mod.THREAD_NAME - @patch("deepctl_cmd_update.plugin_update_check._is_ci", return_value=False) - @patch( - "deepctl_cmd_update.plugin_update_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.plugin_update_check._get_check_interval_seconds", - return_value=86400.0, - ) - def test_fresh_cache_suppresses_thread(self, *_mocks): - """Cache still fresh → no thread should be spawned.""" + def test_failure_waits_a_day(self, check_allowed): + """A failed check is recorded; the next run makes no attempt.""" import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache - mod._thread = None - mod._result = {} + seen = {} - with patch( - "deepctl_cmd_update.plugin_update_check._read_plugin_cache_timestamp", - return_value=time.time(), # just checked + def unreachable(*args, **kwargs): + seen["state"] = check_cache.read(check_cache.PLUGIN_FILE) + raise OSError("unreachable") + + with ( + patch.object( + mod, "_discover_community_plugins", return_value={"deepctl-x": "1"} + ), + patch("httpx.Client.get", side_effect=unreachable), ): check_plugins_and_notify() - assert mod._thread is None + mod._thread.join(timeout=5.0) + # Recorded before the request went out. + assert seen["state"] is not None and seen["state"]["ok"] is False + assert check_cache.read(check_cache.PLUGIN_FILE) is not None + + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() + + def test_success_is_recorded(self, check_allowed): + import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache + + with patch.object(mod, "_discover_community_plugins", return_value={}): + check_plugins_and_notify() + mod._thread.join(timeout=5.0) + state = check_cache.read(check_cache.PLUGIN_FILE) + assert state is not None and state["ok"] is True + assert check_cache.read(check_cache.UPDATE_FILE) is None # --------------------------------------------------------------------------- @@ -436,20 +442,63 @@ def test_no_updates_no_output(self, capsys): captured = capsys.readouterr() assert captured.err == "" - def test_thread_timeout_handling(self): - """If the thread is still running after timeout, no crash.""" + def test_running_check_is_dropped_without_waiting(self): + """A check still running at exit is never joined.""" import deepctl_cmd_update.plugin_update_check as mod - # Create a thread that would take longer than the timeout - mod._result = {} - mod._thread = threading.Thread(target=lambda: time.sleep(10), daemon=True) - mod._thread.start() + release = threading.Event() + mod._result = { + "updates": [{"name": "deepctl-plugin-x", "current": "1", "latest": "2"}] + } + thread = threading.Thread(target=release.wait, daemon=True) + thread.start() + mod._thread = thread + try: + with patch.object(thread, "join") as join: + stderr_capture = io.StringIO() + with patch.object(sys, "stderr", stderr_capture): + print_pending_plugin_notifications() + join.assert_not_called() + assert stderr_capture.getvalue() == "" + assert mod._thread is None + finally: + release.set() + thread.join(timeout=5.0) + - # Should not raise — just returns with no output - with patch.object(mod._thread, "join", side_effect=lambda timeout=None: None): - stderr_capture = io.StringIO() - with patch.object(sys, "stderr", stderr_capture): - print_pending_plugin_notifications() - output = stderr_capture.getvalue() - assert output == "" +class TestPluginAttemptRecording: + @pytest.fixture(autouse=True) + def _reset(self): + import deepctl_cmd_update.plugin_update_check as mod + + mod._thread = None + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + + def test_nothing_recorded_before_the_request(self, check_allowed, monkeypatch): + """A failure before any request (here: discovery) uses up nothing.""" + import deepctl_cmd_update.plugin_update_check as mod + + with patch.object( + mod, "_discover_community_plugins", side_effect=RuntimeError("boom") + ): + check_plugins_and_notify() + mod._thread.join(timeout=5.0) + assert not (check_allowed / "plugin-update-check.json").exists() + + def test_unusable_cache_dir_skips(self, check_allowed, monkeypatch): + import os + + import deepctl_cmd_update.plugin_update_check as mod + from deepctl_cmd_update import check_cache + + check_allowed.mkdir(parents=True, mode=0o755) + monkeypatch.setattr( + check_cache.os, "getuid", lambda: os.stat(check_allowed).st_uid + 1 + ) + with patch.object(mod.threading, "Thread") as thread_cls: + check_plugins_and_notify() + thread_cls.assert_not_called() diff --git a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py index 8d3962c5..4d7b2684 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py @@ -1,22 +1,73 @@ """Unit tests for startup update check.""" +import io +import json +import os +import stat import sys import threading import time -from pathlib import Path from unittest.mock import MagicMock, patch import pytest +from deepctl_cmd_update import check_cache +from deepctl_cmd_update import startup_check as mod from deepctl_cmd_update.startup_check import ( _is_ci, _is_oneshot, - _read_cache_timestamp, - _write_cache_timestamp, check_and_notify, print_pending_notification, ) +_REAL_IS_CI = mod._is_ci +_REAL_IS_AGENT = mod._is_agent + + +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture +def check_allowed(monkeypatch, cache_dir): + """A run where the startup checks may start: an interactive person at a + terminal, outside CI and ``dg mcp``. Returns the cache directory.""" + import deepctl_cmd_update.startup_check as core + + monkeypatch.delenv(core.NO_UPDATE_CHECK_ENV_VAR, raising=False) + monkeypatch.setattr(core, "_is_ci", lambda: False) + monkeypatch.setattr(core, "_stderr_is_terminal", lambda: True) + monkeypatch.setattr(core, "_is_agent", lambda: False) + monkeypatch.setattr(core, "_is_oneshot", lambda: False) + monkeypatch.setattr(core, "_subcommand", lambda: None) + monkeypatch.setattr(core, "_get_check_interval_seconds", lambda quiet: 86400.0) + return cache_dir + + +@pytest.fixture(autouse=True) +def _reset_state(): + mod._thread = None + mod._result = {} + yield + if mod._thread is not None: + mod._thread.join(timeout=5.0) + mod._thread = None + mod._result = {} + + +def _finished_thread() -> threading.Thread: + thread = threading.Thread(target=lambda: None) + thread.start() + thread.join() + return thread + + class TestCIDetection: """Test CI environment detection.""" @@ -48,174 +99,362 @@ def test_travis(self): class TestOneshotDetection: """Test one-shot execution detection.""" - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": ""}, - clear=False, - ) - def test_not_oneshot(self): - import os - os.environ.pop("UV_INTERNAL__PARENT_INTERPRETER", None) + def test_not_oneshot(self, monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("UV_CACHE_DIR", raising=False) + monkeypatch.delenv("PIPX_HOME", raising=False) + monkeypatch.delenv("DEEPCTL_LAUNCHER", raising=False) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "venv")) assert _is_oneshot() is False - @patch.dict( - "os.environ", - {"UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3"}, - clear=True, - ) - def test_uvx_oneshot(self): + def test_uvx_oneshot(self, monkeypatch, tmp_path): + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uvc")) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "uvc" / "archive-v0" / "x")) assert _is_oneshot() is True + def test_npx_oneshot(self, monkeypatch, tmp_path): + main = sys.modules.get("deepctl.main") + if main is not None: + monkeypatch.setattr(main, "LAUNCHER_CAPTURED", False, raising=False) + monkeypatch.setenv("DEEPCTL_LAUNCHER", "npx") + monkeypatch.setattr(sys, "prefix", str(tmp_path / "venv")) + assert _is_oneshot() is True -class TestCacheTimestamp: - """Test file-based timestamp caching.""" - - def test_read_missing_cache(self, tmp_path): - """Missing cache file returns 0.0.""" - with patch( - "deepctl_cmd_update.startup_check._CACHE_FILE", - tmp_path / "nonexistent", - ): - assert _read_cache_timestamp() == 0.0 - - def test_write_and_read_cache(self, tmp_path): - """Written timestamp can be read back.""" - cache_file = tmp_path / "last_version_check" - with patch( - "deepctl_cmd_update.startup_check._CACHE_FILE", cache_file - ), patch( - "deepctl_cmd_update.startup_check._CACHE_DIR", tmp_path - ): - _write_cache_timestamp() - ts = _read_cache_timestamp() - assert ts > 0.0 - assert abs(ts - time.time()) < 5.0 - - -class TestCheckAndNotify: - """Test the startup check lifecycle.""" - @patch.dict("os.environ", {"CI": "true"}, clear=False) - def test_suppressed_in_ci(self): - """No thread is started in CI.""" - import deepctl_cmd_update.startup_check as mod +class TestSkipConditions: + """No check thread starts, and no file is written, when a run skips it.""" - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0") + def _assert_skipped(self, cache_dir): + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() assert mod._thread is None + assert not cache_dir.exists() - @patch( - "deepctl_cmd_update.startup_check._is_ci", return_value=False - ) - @patch( - "deepctl_cmd_update.startup_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.startup_check._get_check_interval_seconds", - return_value=None, - ) - def test_suppressed_when_never(self, *_mocks): - """No thread is started when check_frequency=never.""" - import deepctl_cmd_update.startup_check as mod + def test_starts_when_allowed(self, check_allowed): + with patch.object(mod, "_background_check", lambda ver, res: None): + check_and_notify(current_version="1.0.0") + assert mod._thread is not None + assert mod._thread.daemon is True + assert mod._thread.name == mod.THREAD_NAME + + def test_quiet(self, check_allowed): + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0", quiet=True) + thread_cls.assert_not_called() + + def test_ci(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_ci", lambda: True) + self._assert_skipped(check_allowed) + + def test_ci_env(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_ci", _REAL_IS_CI) + monkeypatch.setenv("CI", "1") + self._assert_skipped(check_allowed) + + def test_agent(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_agent", lambda: True) + self._assert_skipped(check_allowed) + + def test_ai_agent_marker(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_agent", _REAL_IS_AGENT) + monkeypatch.setenv("AI_AGENT", "x") + self._assert_skipped(check_allowed) + + def test_stderr_not_a_terminal(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_stderr_is_terminal", lambda: False) + self._assert_skipped(check_allowed) + + def test_mcp(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_subcommand", lambda: "mcp") + self._assert_skipped(check_allowed) + + @pytest.mark.parametrize("value", ["1", "true", "yes"]) + def test_opt_out_env(self, check_allowed, monkeypatch, value): + monkeypatch.setenv(mod.NO_UPDATE_CHECK_ENV_VAR, value) + self._assert_skipped(check_allowed) + + @pytest.mark.parametrize("value", ["", "0", "false"]) + def test_opt_out_env_off(self, check_allowed, monkeypatch, value): + monkeypatch.setenv(mod.NO_UPDATE_CHECK_ENV_VAR, value) + with patch.object(mod, "_background_check", lambda ver, res: None): + check_and_notify(current_version="1.0.0") + assert mod._thread is not None - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0") - assert mod._thread is None + def test_oneshot(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_is_oneshot", lambda: True) + self._assert_skipped(check_allowed) - def test_suppressed_in_quiet_mode(self): - """No thread is started in quiet mode.""" - import deepctl_cmd_update.startup_check as mod + def test_unusable_cache_dir(self, check_allowed, monkeypatch): + """A foreign-owned cache dir: no check thread, no network, no state.""" + check_allowed.mkdir(parents=True, mode=0o755) + monkeypatch.setattr( + check_cache.os, "getuid", lambda: os.stat(check_allowed).st_uid + 1 + ) + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + assert list(check_allowed.iterdir()) == [] - mod._thread = None - mod._result = {} - check_and_notify(current_version="1.0.0", quiet=True) - assert mod._thread is None + def test_never(self, check_allowed, monkeypatch): + monkeypatch.setattr(mod, "_get_check_interval_seconds", lambda quiet: None) + self._assert_skipped(check_allowed) - @patch.dict("os.environ", {}, clear=False) - @patch( - "deepctl_cmd_update.startup_check._is_ci", return_value=False - ) - @patch( - "deepctl_cmd_update.startup_check._is_oneshot", - return_value=False, - ) - @patch( - "deepctl_cmd_update.startup_check._get_check_interval_seconds", - return_value=86400.0, - ) - @patch( - "deepctl_cmd_update.startup_check._read_cache_timestamp", - return_value=0.0, - ) - @patch( - "deepctl_cmd_update.startup_check._background_check", + +class TestSubcommand: + @pytest.mark.parametrize( + ("args", "expected"), + [ + (["mcp"], "mcp"), + (["-q", "mcp", "--transport", "http"], "mcp"), + (["--profile", "mcp", "models"], "models"), + (["-o", "json", "mcp"], "mcp"), + (["models", "mcp"], "models"), + (["--quiet"], None), + ], ) - def test_thread_starts( - self, - mock_bg_check, - *_mocks, + def test_reads_the_run_args(self, monkeypatch, args, expected): + from deepctl import _cli + + monkeypatch.setattr(_cli, "_run_args", args) + assert mod._subcommand() == expected + + +class TestBackoff: + """The cache file records every attempt, success or not.""" + + def test_failure_is_recorded_and_waits_a_day(self, check_allowed): + with patch("httpx.Client.get", side_effect=OSError("unreachable")): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + + state = json.loads((check_allowed / check_cache.UPDATE_FILE).read_text()) + assert state["ok"] is False + assert state["latest"] is None + assert state["source"] == "pypi" + assert abs(state["checked_at"] - time.time()) < 60 + assert mod._result == {} + + # The next run within the interval makes no attempt. + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + + def test_attempt_recorded_before_connecting(self, check_allowed): + """A check the process never lets finish still counts.""" + seen = {} + + def hang(*args, **kwargs): + path = check_allowed / check_cache.UPDATE_FILE + seen["state"] = json.loads(path.read_text()) + raise OSError("timed out") + + with patch("httpx.Client.get", side_effect=hang): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert seen["state"]["ok"] is False + + def test_nothing_recorded_until_the_client_is_built(self, check_allowed): + """Building the client (CA bundle, proxies) comes before the record.""" + with patch("httpx.Client.__init__", side_effect=OSError("no CA bundle")): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert not (check_allowed / check_cache.UPDATE_FILE).exists() + + def test_run_ending_before_the_request_keeps_the_check_due( + self, check_allowed, monkeypatch ): - """A daemon thread is started when conditions are met.""" - import deepctl_cmd_update.startup_check as mod - - mod._thread = None - mod._result = {} - - # Make _background_check a no-op - mock_bg_check.side_effect = lambda ver, res: None - + """Nothing is recorded until the request is about to go out.""" + monkeypatch.setitem(sys.modules, "httpx", None) # import fails check_and_notify(current_version="1.0.0") - assert mod._thread is not None - assert mod._thread.daemon is True - mod._thread.join(timeout=2.0) + mod._thread.join(timeout=5.0) + assert not (check_allowed / check_cache.UPDATE_FILE).exists() + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + def test_success_is_recorded(self, check_allowed): + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with patch("httpx.Client.get", return_value=response): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + + state = json.loads((check_allowed / check_cache.UPDATE_FILE).read_text()) + assert state["ok"] is True + assert state["latest"] == "2.0.0" + assert mod._result == {"latest": "2.0.0", "current": "1.0.0"} + + def test_due_again_after_the_interval(self, check_allowed): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + path = check_allowed / check_cache.UPDATE_FILE + state = json.loads(path.read_text()) + state["checked_at"] -= 86401 + path.write_text(json.dumps(state)) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + def test_future_timestamp_is_due(self, check_allowed): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + path = check_allowed / check_cache.UPDATE_FILE + state = json.loads(path.read_text()) + state["checked_at"] += 10 * 86400 + path.write_text(json.dumps(state)) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") +class TestCacheFile: + def test_modes(self, cache_dir): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + path = cache_dir / check_cache.UPDATE_FILE + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert [p.name for p in cache_dir.iterdir()] == [check_cache.UPDATE_FILE] + + def test_created_private_under_umask_002(self, cache_dir): + old = os.umask(0o002) + try: + assert check_cache.usable() + finally: + os.umask(old) + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + + @pytest.mark.parametrize("mode", [0o775, 0o770, 0o755, 0o777]) + def test_own_permissive_dir_is_tightened(self, cache_dir, mode): + """Older versions made this directory with a plain mkdir (0775 under umask 002).""" + cache_dir.mkdir(parents=True) + os.chmod(cache_dir, mode) + assert check_cache.usable() + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o700 + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert check_cache.read(check_cache.UPDATE_FILE) is not None + + def test_foreign_owned_dir_is_unusable(self, cache_dir, monkeypatch): + cache_dir.mkdir(parents=True, mode=0o755) + monkeypatch.setattr(check_cache.os, "getuid", lambda: os.stat(cache_dir).st_uid + 1) + assert check_cache.usable() is False + assert stat.S_IMODE(cache_dir.stat().st_mode) == 0o755 # left alone + + def test_tightening_failure_is_unusable(self, cache_dir, monkeypatch): + cache_dir.mkdir(parents=True) + os.chmod(cache_dir, 0o775) + + def refuse(fd, mode): + raise PermissionError("read-only") + + monkeypatch.setattr(check_cache.os, "fchmod", refuse) + assert check_cache.usable() is False + + def test_symlinked_dir_is_unusable(self, cache_dir, tmp_path): + other = tmp_path / "other" + other.mkdir(mode=0o700) + cache_dir.parent.mkdir(parents=True) + cache_dir.symlink_to(other) + assert check_cache.usable() is False + + def test_group_writable_dir_is_never_written_without_tightening(self, cache_dir): + cache_dir.mkdir(parents=True, mode=0o700) + os.chmod(cache_dir, 0o770) + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + assert list(cache_dir.iterdir()) == [] + assert check_cache.read(check_cache.UPDATE_FILE) is None + + def test_group_writable_file_is_not_read(self, cache_dir): + assert check_cache.usable() + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="1", source="pypi") + os.chmod(cache_dir / check_cache.UPDATE_FILE, 0o666) + assert check_cache.read(check_cache.UPDATE_FILE) is None + + def test_symlinked_file_is_not_read(self, cache_dir, tmp_path): + cache_dir.mkdir(parents=True, mode=0o700) + target = tmp_path / "elsewhere.json" + target.write_text(json.dumps({"checked_at": time.time()})) + os.chmod(target, 0o600) + (cache_dir / check_cache.UPDATE_FILE).symlink_to(target) + assert check_cache.read(check_cache.UPDATE_FILE) is None + + @pytest.mark.parametrize("data", ["not json", "[]", '{"checked_at": true}']) + def test_bad_contents_read_as_never_checked(self, cache_dir, data): + cache_dir.mkdir(parents=True, mode=0o700) + path = cache_dir / check_cache.UPDATE_FILE + path.write_text(data) + os.chmod(path, 0o600) + assert check_cache.due(check_cache.UPDATE_FILE, 86400.0) + + +class TestNoConfigWrite: + def test_check_never_saves_the_config(self, check_allowed): + from deepctl_core import Config + + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with ( + patch.object(Config, "save") as save, + patch.object(Config, "_set_config_value") as set_value, + patch("httpx.Client.get", return_value=response), + ): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + save.assert_not_called() + set_value.assert_not_called() class TestPrintPendingNotification: """Test notification display.""" def test_no_notification_when_no_thread(self, capsys): - """No output when no thread was started.""" - import deepctl_cmd_update.startup_check as mod - - mod._thread = None - mod._result = {} print_pending_notification() captured = capsys.readouterr() assert captured.err == "" def test_notification_displayed(self): """Notification is printed to stderr when update found.""" - import deepctl_cmd_update.startup_check as mod - - # Simulate completed thread with result mod._result = {"latest": "2.0.0", "current": "1.0.0"} - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() - - # Capture stderr - import io + mod._thread = _finished_thread() stderr_capture = io.StringIO() with patch.object(sys, "stderr", stderr_capture): print_pending_notification() output = stderr_capture.getvalue() + assert output.count("Update available") == 1 assert "2.0.0" in output assert "1.0.0" in output assert "dg update" in output + # Printed once: a second call has nothing to show. + stderr_capture = io.StringIO() + with patch.object(sys, "stderr", stderr_capture): + print_pending_notification() + assert stderr_capture.getvalue() == "" + + def test_running_check_is_dropped_without_waiting(self): + """A check still running at exit is never joined.""" + release = threading.Event() + mod._result = {"latest": "2.0.0", "current": "1.0.0"} + mod._thread = threading.Thread(target=release.wait, daemon=True) + mod._thread.start() + thread = mod._thread + try: + with patch.object(thread, "join") as join: + stderr_capture = io.StringIO() + started = time.monotonic() + with patch.object(sys, "stderr", stderr_capture): + print_pending_notification() + elapsed = time.monotonic() - started + join.assert_not_called() + assert stderr_capture.getvalue() == "" + assert elapsed < 2.0 + assert mod._thread is None + finally: + release.set() + thread.join(timeout=5.0) + def test_no_notification_when_up_to_date(self, capsys): """No output when no update is available.""" - import deepctl_cmd_update.startup_check as mod - mod._result = {} # No update found - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() + mod._thread = _finished_thread() print_pending_notification() captured = capsys.readouterr() @@ -230,15 +469,99 @@ def test_no_notification_when_up_to_date(self, capsys): ) def test_broken_pipe_swallowed(self, error): """A closed/broken stderr (e.g. `dg mcp` host disconnect) is tolerated.""" - import deepctl_cmd_update.startup_check as mod - mod._result = {"latest": "2.0.0", "current": "1.0.0"} - mod._thread = threading.Thread(target=lambda: None) - mod._thread.start() - mod._thread.join() + mod._thread = _finished_thread() broken_stderr = MagicMock() broken_stderr.write.side_effect = error with patch.object(sys, "stderr", broken_stderr): # Must not raise. print_pending_notification() + + +class TestSavedNotice: + """An answer that came too late for its run gives the notice later.""" + + @staticmethod + def _save(cache_dir, **state): + assert check_cache.usable() + path = cache_dir / check_cache.UPDATE_FILE + path.write_text(json.dumps(state)) + os.chmod(path, 0o600) + + @staticmethod + def _notice() -> str: + err = io.StringIO() + with patch.object(sys, "stderr", err): + print_pending_notification() + return err.getvalue() + + def test_printed_once_per_interval(self, check_allowed): + self._save(check_allowed, checked_at=time.time(), ok=True, latest="2.0.0") + with patch.object(mod.threading, "Thread") as thread_cls: + check_and_notify(current_version="1.0.0") + thread_cls.assert_not_called() + out = self._notice() + assert out.count("Update available") == 1 and "2.0.0" in out + state = check_cache.read(check_cache.UPDATE_FILE) + assert abs(check_cache.notified_at(state) - time.time()) < 60 + + check_and_notify(current_version="1.0.0") + assert self._notice() == "" + + def test_again_after_the_interval(self, check_allowed): + self._save( + check_allowed, + checked_at=time.time(), + ok=True, + latest="2.0.0", + notified_at=time.time() - 86401, + ) + check_and_notify(current_version="1.0.0") + assert "2.0.0" in self._notice() + + @pytest.mark.parametrize( + "state", + [ + {"ok": True, "latest": "1.0.0"}, + {"ok": True, "latest": "0.9.0"}, + {"ok": False, "latest": None}, + {"ok": True, "latest": "not a version"}, + ], + ) + def test_nothing_newer_no_notice(self, check_allowed, state): + self._save(check_allowed, checked_at=time.time(), **state) + check_and_notify(current_version="1.0.0") + assert self._notice() == "" + + def test_late_answer_reaches_the_next_run(self, check_allowed): + """A sub-second command drops the result; the second run shows it.""" + release = threading.Event() + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + + def slow_get(*args, **kwargs): + release.wait(5.0) + return response + + with patch("httpx.Client.get", side_effect=slow_get): + check_and_notify(current_version="1.0.0") + thread = mod._thread + assert self._notice() == "" # run 1 ends before the answer + release.set() + thread.join(timeout=5.0) + + check_and_notify(current_version="1.0.0") # run 2: not due + assert mod._thread is None + out = self._notice() + assert out.count("Update available") == 1 and "1.0.0 → 2.0.0" in out + + def test_in_time_notice_is_not_repeated_from_the_cache(self, check_allowed): + response = MagicMock() + response.json.return_value = {"info": {"version": "2.0.0"}} + with patch("httpx.Client.get", return_value=response): + check_and_notify(current_version="1.0.0") + mod._thread.join(timeout=5.0) + assert "2.0.0" in self._notice() + check_and_notify(current_version="1.0.0") + assert self._notice() == "" diff --git a/packages/deepctl-cmd-update/tests/unit/test_update_command.py b/packages/deepctl-cmd-update/tests/unit/test_update_command.py index 3a89f6da..924403a1 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_update_command.py +++ b/packages/deepctl-cmd-update/tests/unit/test_update_command.py @@ -1,5 +1,6 @@ """Unit tests for update command.""" +import os import subprocess from unittest.mock import MagicMock, patch @@ -9,6 +10,55 @@ from deepctl_cmd_update.version_check import VersionInfo +def _real_config(tmp_path, monkeypatch): + """The default user config (so update.* keys are honored), in tmp_path.""" + from deepctl_core import Config + + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + path = tmp_path / "xdg" / "deepctl" / "config.yaml" + path.parent.mkdir(parents=True, mode=0o700) + path.write_text( + "profiles:\n default:\n project_id: P1\n" + "update:\n installation_method: pipx\n check_frequency: daily\n" + ) + os.chmod(path, 0o600) + config = Config() + assert config.config_path == path + return config, path + + +class TestConfigUntouched: + """dg update never writes the user config (no implicit save).""" + + @pytest.mark.parametrize("returncode", [0, 1]) + @patch("deepctl_cmd_update.command.subprocess.run") + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + @patch("deepctl_cmd_update.command.asyncio.run") + def test_full_update_leaves_the_file_byte_identical( + self, run_check, detect, run, returncode, tmp_path, monkeypatch + ): + config, path = _real_config(tmp_path, monkeypatch) + before = (path.read_bytes(), path.stat().st_mtime_ns) + run_check.return_value = VersionInfo( + current_version="0.1.0", latest_version="0.2.0", update_available=True + ) + detect.return_value = InstallationInfo( + method=InstallMethod.PIP, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + run.return_value = subprocess.CompletedProcess([], returncode, "", "") + UpdateCommand().handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), yes=True + ) + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + # The legacy key is still readable, and was not rewritten. + assert config.get("update.installation_method") == "pipx" + + class TestUpdateCommand: """Test update command functionality.""" @@ -537,3 +587,120 @@ def test_failed_version_check_exits_one( assert command.exit_code_for(result) == 1 assert result.status == "error" assert "connection refused" in result.message + + +class TestDryRun: + """`dg update --dry-run` reports the method and command, and nothing else.""" + + @pytest.mark.parametrize( + ("method", "expected"), + [ + (InstallMethod.PIPX, "Run: pipx upgrade deepctl"), + (InstallMethod.UV_TOOL, "Run: uv tool install deepctl@latest"), + (InstallMethod.HOMEBREW, "Run: brew upgrade deepgram"), + ], + ) + @patch("deepctl_cmd_update.command.VersionChecker") + @patch("deepctl_cmd_update.command.subprocess.run") + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_prints_the_command( + self, detect, run, checker, method, expected, capsys, monkeypatch + ): + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "format", "default") + monkeypatch.setitem(output._output_config, "quiet", False) + detect.return_value = InstallationInfo( + method=method, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + config = MagicMock() + command = UpdateCommand() + result = command.handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert result["message"] == expected + assert result["installation_method"] == method.value + assert command.exit_code_for(result) == 0 + out = capsys.readouterr().out + assert expected in out + assert f"Installation method: {method.value}" in out + checker.assert_not_called() + run.assert_not_called() + config.save.assert_not_called() + config._set_config_value.assert_not_called() + + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_dry_run_leaves_the_config_byte_identical( + self, detect, tmp_path, monkeypatch + ): + config, path = _real_config(tmp_path, monkeypatch) + before = (path.read_bytes(), path.stat().st_mtime_ns) + detect.return_value = InstallationInfo( + method=InstallMethod.PIPX, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + UpdateCommand().handle( + config=config, auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + + @patch("deepctl_cmd_update.command.InstallationDetector.detect") + def test_json_keeps_stdout_for_the_payload(self, detect, capsys, monkeypatch): + from deepctl_core import output + + monkeypatch.setitem(output._output_config, "format", "json") + detect.return_value = InstallationInfo( + method=InstallMethod.PIPX, + path="/p", + virtual_env=True, + editable=False, + python_executable="/p/bin/python3", + ) + result = UpdateCommand().handle( + config=MagicMock(), auth_manager=MagicMock(), client=MagicMock(), dry_run=True + ) + assert result["message"] == "Run: pipx upgrade deepctl" + assert capsys.readouterr().out == "" + + +class TestUnreachableIndex: + @patch("deepctl_cmd_update.command.asyncio.run") + @patch("deepctl_cmd_update.command.VersionChecker") + def test_raises_update_check_failed(self, checker, run): + from deepctl_cmd_update.version_check import UpdateCheckError + from deepctl_core.transport import TransportError + + run.side_effect = UpdateCheckError("pypi.org") + with pytest.raises(TransportError) as raised: + UpdateCommand().handle( + config=MagicMock(), auth_manager=MagicMock(), client=MagicMock() + ) + err = raised.value + assert err.code == "update_check_failed" + assert err.details == {"host": "pypi.org"} + assert err.error == "Couldn't check for updates: couldn't reach pypi.org." + + def test_renders_with_both_hints(self, monkeypatch): + from deepctl_core import output, transport + + err = transport.TransportError( + "update_check_failed", values={"host": "pypi.org"}, details={"host": "pypi.org"} + ) + monkeypatch.setitem(output._output_config, "agentic", False) + human = transport.render(err, command_path="update") + assert human.code == "update_check_failed" + assert human.hint.startswith("Check your network connection") + assert human.next_step is None + assert human.details == {"host": "pypi.org"} + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setenv("AI_AGENT", "x") + agent = transport.render(err, command_path="update") + assert agent.hint.startswith("The update check needs network access") diff --git a/packages/deepctl-cmd-update/tests/unit/test_version_check.py b/packages/deepctl-cmd-update/tests/unit/test_version_check.py index f96f4fc6..2954d988 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_version_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_version_check.py @@ -14,6 +14,25 @@ ) +@pytest.fixture +def cache_dir(tmp_path, monkeypatch): + """dg's cache directory, pointed at a fresh temp dir (not created yet).""" + from deepctl import _cache + + root = tmp_path / "cache" / "deepctl" + monkeypatch.setattr(_cache, "cache_root", lambda: str(root)) + return root + + +@pytest.fixture(autouse=True) +def _isolated_cache(cache_dir): + """Every test here gets its own empty cache directory.""" + from deepctl_cmd_update import check_cache + + assert check_cache.usable() + return cache_dir + + class TestCheckFrequency: """Test check_frequency configuration values.""" @@ -274,3 +293,123 @@ def test_update_available(self): assert "1.0.0" in msg assert "2.0.0" in msg assert "dg update" in msg + + +class TestCacheFile: + """The last answer lives in the cache file, never in the user config.""" + + @staticmethod + def _config(values=None): + config = MagicMock() + values = {"update.check_enabled": True, **(values or {})} + config.get.side_effect = lambda key, default=None: values.get(key, default) + return config + + @pytest.mark.asyncio + async def test_answer_goes_to_the_cache_file(self, cache_dir): + from deepctl_cmd_update import check_cache + + config = self._config() + checker = VersionChecker(config, "1.0.0") + info = VersionInfo( + current_version="1.0.0", latest_version="2.0.0", update_available=True + ) + with patch.object(checker, "_check_pypi", AsyncMock(return_value=info)): + await checker.check_version(force=True) + + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None + assert (state["ok"], state["latest"], state["source"]) == ( + True, + "2.0.0", + "pypi", + ) + config.save.assert_not_called() + config._set_config_value.assert_not_called() + + @pytest.mark.asyncio + async def test_github_answer_names_its_source(self): + from deepctl_cmd_update import check_cache + + checker = VersionChecker(self._config(), "1.0.0") + response = MagicMock() + response.json.return_value = {"tag_name": "v2.0.0", "html_url": "u"} + client = AsyncMock() + client.get.return_value = response + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + with patch("httpx.AsyncClient", return_value=client): + await checker.check_version(force=True, install_method="homebrew") + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None and state["source"] == "github" + + def test_cache_file_wins_over_legacy_keys(self): + from deepctl_cmd_update import check_cache + + check_cache.write(check_cache.UPDATE_FILE, ok=True, latest="3.0.0", source="pypi") + config = self._config( + { + "update.last_check": "2000-01-01T00:00:00+00:00", + "update.cached_version_info": { + "current_version": "1.0.0", + "latest_version": "9.9.9", + "update_available": True, + }, + } + ) + checker = VersionChecker(config, "1.0.0") + assert checker.should_check() is False + info = checker._get_cached_info() + assert info is not None + assert (info.latest_version, info.update_available) == ("3.0.0", True) + + def test_failed_check_has_no_cached_answer(self): + from deepctl_cmd_update import check_cache + + check_cache.write(check_cache.UPDATE_FILE, ok=False, latest=None, source="pypi") + checker = VersionChecker(self._config(), "1.0.0") + assert checker.should_check() is False + assert checker._get_cached_info() is None + + def test_legacy_keys_read_when_no_cache_file(self): + config = self._config( + { + "update.cached_version_info": { + "current_version": "1.0.0", + "latest_version": "1.5.0", + "update_available": True, + }, + } + ) + info = VersionChecker(config, "1.0.0")._get_cached_info() + assert info is not None and info.latest_version == "1.5.0" + + +class TestUnreachable: + """No answer is an error, never "you are on the latest version".""" + + @staticmethod + def _config(): + config = MagicMock() + config.get.side_effect = lambda key, default=None: ( + True if key == "update.check_enabled" else default + ) + return config + + @pytest.mark.asyncio + @pytest.mark.parametrize("install_method", [None, "homebrew"]) + async def test_raises_and_records_the_failure(self, install_method): + from deepctl_cmd_update import check_cache + from deepctl_cmd_update.version_check import UpdateCheckError + + client = AsyncMock() + client.get.side_effect = httpx.ConnectError("unreachable") + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + checker = VersionChecker(self._config(), "1.0.0") + with patch("httpx.AsyncClient", return_value=client): + with pytest.raises(UpdateCheckError) as raised: + await checker.check_version(force=True, install_method=install_method) + assert raised.value.host == "pypi.org" + state = check_cache.read(check_cache.UPDATE_FILE) + assert state is not None and state["ok"] is False diff --git a/packages/deepctl-core/src/deepctl_core/auth.py b/packages/deepctl-core/src/deepctl_core/auth.py index 03044105..ac1506dc 100644 --- a/packages/deepctl-core/src/deepctl_core/auth.py +++ b/packages/deepctl-core/src/deepctl_core/auth.py @@ -3,14 +3,15 @@ import os import time import webbrowser +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar, cast from urllib.parse import urlsplit from pydantic import BaseModel, field_validator -from . import hosts, output +from . import credentials, hosts, output, transport from . import messages as msg from .client import _split_base_url from .config import Config @@ -24,12 +25,16 @@ import httpx # noqa: TC004 import keyring # noqa: TC004 -# httpx and keyring load when the first AuthManager is built, not when this -# module is imported (every command module imports it). _load_modules() -# binds them as module globals; __getattr__ makes `deepctl_core.auth.keyring` -# the real module before that, so tests can patch it as before. +# httpx and keyring load when they're first used, not when this module is +# imported (every command module imports it) or an AuthManager is built +# (every command builds one). __getattr__ binds them as module globals on +# first access, so `deepctl_core.auth.keyring` is the real module and tests +# can patch it as before. The keyring is read through +# deepctl_core.credentials, which bounds the read off a terminal. _LAZY_MODULES = ("httpx", "keyring") +_F = TypeVar("_F", bound=Callable[..., Any]) + def _bind_module(name: str) -> Any: import importlib @@ -48,8 +53,8 @@ def __getattr__(name: str) -> Any: return _bind_module(name) -def _load_modules() -> None: - for name in _LAZY_MODULES: +def _load_modules(*names: str) -> None: + for name in names or _LAZY_MODULES: if name not in globals(): _bind_module(name) @@ -162,6 +167,7 @@ def _insecure_mounts() -> "dict[str, httpx.BaseTransport]": httpx builds from HTTPS_PROXY/ALL_PROXY/NO_PROXY, so every other host keeps both certificate verification and the proxy settings. """ + _load_modules("httpx") unverified = httpx.HTTPTransport(verify=False) return { f"all://{'[::1]' if host == '::1' else host}": unverified @@ -205,6 +211,54 @@ def __init__(self, host: str) -> None: self.host = host +def _forgets_resolved(method: _F) -> _F: + """Forget cached credential answers once ``method`` has written the keyring + or a profile (also when it fails part-way), so later questions see the + new state.""" + import functools + + @functools.wraps(method) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + try: + return method(self, *args, **kwargs) + finally: + self._forget_resolved() + + return cast("_F", wrapper) + + +class AuthRequiredError(transport.TransportError, AuthenticationError): + """``auth_required``: no API key is set (``E-AUTH-REQUIRED``), or the only + one was in a user config file this AI agent session doesn't trust + (``E-KEY-UNTRUSTED``). + + A :class:`~deepctl_core.transport.TransportError`, so ``BaseCommand`` + renders the coded result, and an :class:`AuthenticationError`, so code + that caught ``guard()``'s error before 0.5.0 still does. + """ + + def __init__(self, untrusted_source: str | None = None) -> None: + if untrusted_source is None: + super().__init__("auth_required", {}, {}) + else: + super().__init__( + "auth_required", + {"user_config": untrusted_source}, + {}, + transport.KEY_UNTRUSTED, + ) + + +class _NoFlags: + """A credentials context without ``--api-key`` and ``--project-id``.""" + + explicit_api_key = None + explicit_project_id = None + + def __init__(self, config: Config) -> None: + self.config = config + + class AuthManager: """Cross-platform authentication manager.""" @@ -221,23 +275,103 @@ def __init__( explicit_api_key: Explicitly provided API key (e.g., from CLI flags) explicit_project_id: Explicitly provided project ID (e.g., from CLI flags) """ - _load_modules() self.config = config self.explicit_api_key = explicit_api_key self.explicit_project_id = explicit_project_id - # DEEPGRAM_CLI_INSECURE=1 skips TLS verification for localhost, - # 127.0.0.1 and [::1] (local dev), never for any other host and never - # in an AI agent session: the agent may have set it. Proxy settings - # from the environment still apply to every other host. - if _insecure_allowed(): - self.client = httpx.Client( - timeout=30.0, - verify=True, - mounts=_insecure_mounts(), - event_hooks={"request": [_warn_if_verified]}, + # One credential resolution per question set (deepctl_core.credentials); + # forgotten whenever this manager writes the keyring or a profile. + self._resolved: dict[tuple[Any, ...], credentials.Resolved] = {} + # Keyring entries this manager already removed (expired credentials). + self._pruned: set[str] = set() + self._client: httpx.Client | None = None + + @property + def client(self) -> "httpx.Client": + """The HTTP client for login's device flow and key verification. + + Built on first use, so no other command makes a second client; API + requests use the shared one (``deepctl_core.transport``). + """ + if self._client is None: + _load_modules("httpx") + # DEEPGRAM_CLI_INSECURE=1 skips TLS verification for localhost, + # 127.0.0.1 and [::1] (local dev), never for any other host and + # never in an AI agent session: the agent may have set it. Proxy + # settings from the environment still apply to every other host. + if _insecure_allowed(): + self._client = httpx.Client( + timeout=30.0, + verify=True, + mounts=_insecure_mounts(), + event_hooks={"request": [_warn_if_verified]}, + ) + else: + self._client = httpx.Client(timeout=30.0, verify=True) + return self._client + + @client.setter + def client(self, value: "httpx.Client | None") -> None: + self._client = value + + @client.deleter + def client(self) -> None: + self._client = None + + def resolve( + self, + *, + profile: str | None = None, + ignore_env: bool = False, + need_key: bool = True, + use_flags: bool = True, + prune: bool = True, + ) -> credentials.Resolved: + """This run's credentials and their sources (``credentials.resolve``). + + ``use_flags=False`` answers as if no ``--api-key`` or + ``--project-id`` had been given (the stored-profile questions). + ``prune=False`` keeps it read-only (introspection: whoami, login's + checks); answers are cached either way, so one run resolves once. + """ + ctx: Any = self if use_flags else _NoFlags(self.config) + name = profile or credentials.profile_name(self.config)[0] + stored = self.config.get_profile(name) + state = ( + name, + ignore_env, + ctx.explicit_api_key, + ctx.explicit_project_id, + stored.api_key, + stored.project_id, + stored.dg_token_expires_at, + ) + # A full answer also answers the project-only question. + found = self._resolved.get((True, *state)) + if found is None and not need_key: + found = self._resolved.get((False, *state)) + if found is None: + found = credentials.resolve( + ctx, + profile=profile, + ignore_env=ignore_env, + need_key=need_key, + prune=False, ) - else: - self.client = httpx.Client(timeout=30.0, verify=True) + self._resolved[(need_key, *state)] = found + # Project-only questions (need_key=False) never touch the keyring. + if ( + prune + and need_key + and found.prune_target + and found.prune_target not in self._pruned + ): + self._pruned.add(found.prune_target) + credentials.prune_expired(found) + return found + + def _forget_resolved(self) -> None: + self._resolved.clear() + self._pruned.clear() def check_base_url(self) -> str: """Return the base URL credentials may be sent to. @@ -253,8 +387,8 @@ def has_env_credentials(self) -> tuple[bool, bool]: Returns: Tuple of (has_api_key, has_project_id) """ - return bool(os.getenv("DEEPGRAM_API_KEY")), bool( - os.getenv("DEEPGRAM_PROJECT_ID") + return bool(credentials.env_value("DEEPGRAM_API_KEY")), bool( + credentials.env_value("DEEPGRAM_PROJECT_ID") ) def has_profile_credentials( @@ -262,35 +396,17 @@ def has_profile_credentials( ) -> tuple[bool, bool]: """Check if profile has stored credentials. + As in 0.4.2, a profile key is a usable keyring entry or a key in the + profile as the config holds it (which includes ``DEEPGRAM_API_KEY``). + Args: profile_name: Profile to check (uses current profile if not specified) Returns: Tuple of (has_api_key, has_project_id) """ - profile_name = profile_name or self.config.profile or "default" - - # Check keyring for a direct API key. - has_api_key = False - expired, _ = self._credential_expiry(profile_name) - if not expired: - try: - has_api_key = bool( - keyring.get_password(KEYRING_SERVICE, f"api-key.{profile_name}") - ) - except Exception: - pass - - # If not in keyring, check config. - if not has_api_key: - profile = self.config.get_profile(profile_name) - has_api_key = bool(profile.api_key) - - # Check for project ID in profile - profile = self.config.get_profile(profile_name) - has_project_id = bool(profile.project_id) - - return has_api_key, has_project_id + resolved = self.resolve(profile=profile_name, use_flags=False, prune=False) + return resolved.profile_has_key, resolved.profile_has_project def is_authenticated(self, check_profile_only: bool = False) -> bool: """Check if user is authenticated. @@ -298,24 +414,23 @@ def is_authenticated(self, check_profile_only: bool = False) -> bool: Args: check_profile_only: If True, only check profile credentials, not env vars """ - # Check explicit credentials first (highest priority) if self.explicit_api_key: return True - - # Check profile credentials - profile_name = self.config.profile or "default" - has_profile_key, _ = self.has_profile_credentials(profile_name) - if has_profile_key: + resolved = self.resolve(use_flags=False, prune=False) + if resolved.profile_has_key: return True - - # Check environment variables (unless checking profile only) - return not check_profile_only and bool(os.getenv("DEEPGRAM_API_KEY")) + return not check_profile_only and bool( + credentials.env_value("DEEPGRAM_API_KEY") + ) def is_ci_mode(self) -> bool: """Check if running in CI mode (credentials from environment).""" # If both API key and project ID are provided via environment, # we're in CI mode - return bool(os.getenv("DEEPGRAM_API_KEY") and os.getenv("DEEPGRAM_PROJECT_ID")) + return bool( + credentials.env_value("DEEPGRAM_API_KEY") + and credentials.env_value("DEEPGRAM_PROJECT_ID") + ) def get_api_key(self, ignore_env: bool = False) -> str | None: """Get API key following precedence: explicit > profile > env. @@ -327,74 +442,20 @@ def get_api_key(self, ignore_env: bool = False) -> str | None: Args: ignore_env: If True, don't check environment variables """ - # Explicit credentials (--api-key flag, CI/CD) bypass the JWT flow. - if self.explicit_api_key: - return self.explicit_api_key - - profile_name = self.config.profile or "default" - current_profile = self.config.get_profile(profile_name) - - expired, expiry_error = self._credential_expiry(profile_name) - if not expired: - # Profile keyring entry, followed by its config-file fallback. - try: - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) - if api_key: - return api_key - except Exception: - pass - - if current_profile.api_key: - return current_profile.api_key - else: - self._delete_keyring_credential(f"api-key.{profile_name}") - - # Environment variable — lowest priority. - if not ignore_env: - api_key = os.getenv("DEEPGRAM_API_KEY") - if api_key: - return api_key - - if expiry_error: - raise AuthenticationError(expiry_error) - + resolved = self.resolve(ignore_env=ignore_env) + if resolved.api_key: + return resolved.api_key + if resolved.expiry_error: + raise AuthenticationError(resolved.expiry_error) return None def _credential_expiry(self, profile_name: str) -> tuple[bool, str | None]: """Return whether a finite profile credential is unusable and why.""" - expires_at = self.config.get_profile(profile_name).dg_token_expires_at - if not isinstance(expires_at, str): - return False, None - - try: - parsed_expiry = datetime.fromisoformat(expires_at) - if parsed_expiry.tzinfo is None: - raise ValueError - except (TypeError, ValueError): - return ( - True, - "Your Deepgram credential expiry is invalid. Run `dg login` again.", - ) - - if parsed_expiry <= datetime.now(timezone.utc): - return True, "Your Deepgram credential has expired. Run `dg login` again." - - return False, None + return credentials.credential_expiry(self.config, profile_name) def _delete_keyring_credential(self, keyring_key: str) -> None: """Remove a credential when present without masking keyring failures.""" - from keyring.errors import PasswordDeleteError - - try: - keyring.delete_password(KEYRING_SERVICE, keyring_key) - except PasswordDeleteError: - pass - except Exception as exc: - console.print( - f"[yellow]Warning:[/yellow] Could not remove legacy credential: {exc}" - ) + credentials.keyring_delete(keyring_key) def get_project_id(self, ignore_env: bool = False) -> str | None: """Get project ID following precedence: explicit > profile > env. @@ -402,23 +463,7 @@ def get_project_id(self, ignore_env: bool = False) -> str | None: Args: ignore_env: If True, don't check environment variables """ - # Explicit credentials have highest priority - if self.explicit_project_id: - return self.explicit_project_id - - # Check profile next - profile_name = self.config.profile or "default" - current_profile = self.config.get_profile(profile_name) - if current_profile.project_id: - return current_profile.project_id - - # Environment variable has lowest priority - if not ignore_env: - project_id = os.getenv("DEEPGRAM_PROJECT_ID") - if project_id: - return project_id - - return None + return self.resolve(ignore_env=ignore_env, need_key=False).project_id def get_credential_source(self) -> str: """Get a description of where credentials are coming from. @@ -429,16 +474,18 @@ def get_credential_source(self) -> str: if self.explicit_api_key or self.explicit_project_id: return "explicit flags" - profile_name = self.config.profile or "default" - has_profile_key, has_profile_project = self.has_profile_credentials( - profile_name - ) - - if has_profile_key or has_profile_project: - return f"profile '{profile_name}'" - - has_env_key, has_env_project = self.has_env_credentials() - if has_env_key or has_env_project: + # The real source: a profile key equal to DEEPGRAM_API_KEY came from + # the environment (Config merges it into the profile). + resolved = self.resolve() + if resolved.key_source in ( + credentials.KEY_SOURCE_KEYRING, + credentials.KEY_SOURCE_FILE, + ) or (resolved.project_source == "profile"): + return f"profile '{resolved.profile}'" + if ( + resolved.key_source == credentials.KEY_SOURCE_ENV + or resolved.project_source == "env" + ): return "environment variables" return "no credentials" @@ -454,6 +501,7 @@ def verify_api_key( Returns: Tuple of (success, message, error_type) """ + _load_modules("httpx") if not api_key: api_key = self.get_api_key() @@ -467,7 +515,7 @@ def verify_api_key( try: headers = { - "Authorization": f"Token {api_key}", + **transport.request_headers(api_key), "Content-Type": "application/json", } @@ -520,6 +568,7 @@ def verify_credentials( - error_type: 'auth' for API key issues, 'project' for project ID issues, None if successful """ + _load_modules("httpx") # Use provided credentials or get from storage if not api_key: api_key = self.get_api_key() @@ -539,7 +588,7 @@ def verify_credentials( # Make API request to verify both credentials try: headers = { - "Authorization": f"Token {api_key}", + **transport.request_headers(api_key), "Content-Type": "application/json", } @@ -582,38 +631,26 @@ def guard(self) -> None: """Check that an API key is available, without sending it anywhere. Refuses first, before reading any key, when the base URL isn't - trusted with credentials. The key itself is checked by the command's - own request: a 401 or 403 there becomes ``auth_invalid`` or - ``auth_forbidden`` (see ``deepctl_core.transport``). Project ID - validation is handled separately by requires_project on BaseCommand. + trusted with credentials. With no key, raises :class:`AuthRequiredError` + (``auth_required``), which ``BaseCommand`` renders; the key itself + is checked by the command's own request: a 401 or 403 there becomes + ``auth_invalid`` or ``auth_forbidden`` (see ``deepctl_core.transport``). + Project ID validation is handled separately by requires_project on + BaseCommand. """ self.check_base_url() - api_key = self.get_api_key() - - if not api_key and getattr(self.config, "api_key_untrusted", False) is True: + # Raises the expiry error when a stored credential expired and + # nothing else is set, as before. + if self.get_api_key(): + return + resolved = self.resolve() + if resolved.untrusted_source: # The file's key was left unused (W-PERM-AGENT / W-CFG-UNTRUSTED): # say so, and send the agent to a person, not to dg login. - from rich.markup import escape - - text = msg.E_KEY_UNTRUSTED.format( - user_config=msg.display_path(self.config.config_path) - ) - output.print_error(escape(text)) - output.print_hint(msg.W_CFG_UNTRUSTED_FILE_HINT) - raise AuthenticationError(text) - if not api_key: - console.print( - "[red]Error:[/red] DEEPGRAM_API_KEY is not set in the " - "configuration file " - f"({self.config.config_path}) or environment variable.\n" - ) - console.print( - "[yellow]Run[/yellow] [bold]deepctl login[/bold] " - "[yellow]to configure the CLI with your Deepgram " - "account.[/yellow]\n" - ) - raise AuthenticationError("DEEPGRAM_API_KEY is not set") + raise AuthRequiredError(untrusted_source=resolved.untrusted_source) + raise AuthRequiredError() + @_forgets_resolved def login_with_api_key( self, api_key: str, project_id: str | None = None, _force_write: bool = False ) -> None: @@ -651,6 +688,7 @@ def login_with_api_key( console.print(f"[green]✓[/green] {message}") # Store API key in keyring for security + _load_modules("keyring") profile_name = self.config.profile or "default" keyring_available = False # Nothing goes to the keyring when the profile can't be saved. @@ -753,6 +791,7 @@ def _request_device_code( Returns: DeviceCodeResponse with device code and verification URI """ + _load_modules("httpx") payload = { "client_id": CLIENT_ID, "hostname": DEVICE_NAME, @@ -796,6 +835,7 @@ def poll_device_token( and no device name is sent. Network errors are raised (``httpx.RequestError``). """ + _load_modules("httpx") base = identity_url or check_identity_host(self.config) response = self.client.post( f"{base}/token", @@ -845,6 +885,7 @@ def _poll_for_token( identity_url: str | None = None, ) -> TokenResponse: """Poll for token using device code until approved, denied or expired.""" + _load_modules("httpx") console.print("\n[blue]Waiting for authentication...[/blue]") start_time = time.time() @@ -886,6 +927,7 @@ def _poll_for_token( raise AuthenticationError("Authentication timed out") + @_forgets_resolved def _store_token(self, token_response: TokenResponse) -> None: """Store authentication token. @@ -893,6 +935,7 @@ def _store_token(self, token_response: TokenResponse) -> None: expires independently of the OIDC access token and requires a new device authorization when it expires. """ + _load_modules("keyring") profile_name = self.config.profile or "default" if token_response.dg_token: @@ -944,6 +987,7 @@ def _store_token(self, token_response: TokenResponse) -> None: "Deepgram sign-in did not return a finite credential. Run `dg update` and `dg login` again." ) + @_forgets_resolved def logout(self, keep_config: bool = False) -> None: """Logout user and clear credentials. @@ -951,6 +995,7 @@ def logout(self, keep_config: bool = False) -> None: keep_config: If True, keep profile configuration (project ID, base URL) and only clear credentials. If False, remove profile completely. """ + _load_modules("keyring") profile_name = self.config.profile or "default" # Nothing is cleared from the keyring when the profile can't be saved. self.config.check_saveable() @@ -992,9 +1037,8 @@ def list_profiles(self) -> ProfilesResult: project_id = profile.project_id try: - api_key = keyring.get_password( - KEYRING_SERVICE, f"api-key.{profile_name}" - ) + # Bounded off a terminal; a timeout falls back like an error. + api_key = credentials.keyring_read(f"api-key.{profile_name}") # Project ID is only stored in config, not keyring except Exception: # Fall back to config @@ -1017,5 +1061,6 @@ def list_profiles(self) -> ProfilesResult: def __del__(self) -> None: """Cleanup HTTP client.""" - if hasattr(self, "client"): - self.client.close() + client = self.__dict__.get("_client") + if client is not None: + client.close() diff --git a/packages/deepctl-core/src/deepctl_core/base_command.py b/packages/deepctl-core/src/deepctl_core/base_command.py index 1b930b31..914e6dca 100644 --- a/packages/deepctl-core/src/deepctl_core/base_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_command.py @@ -6,6 +6,7 @@ import click +from . import credentials from .auth import AuthManager from .client import DeepgramClient from .config import Config, flush_config_notices @@ -153,6 +154,7 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: with TimingContext(f"command_{self.name}_total"): # Output mode is set up by now; print queued config notices. flush_config_notices() + credentials.flush_warnings() with TimingContext("command_setup"): # Get configuration from context config = ctx.obj.get("config") @@ -172,7 +174,7 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: ) # Create Deepgram client - client = DeepgramClient(config, auth_manager) + client = self._make_client(config, auth_manager) # Refusals that need no credentials come before the auth check, # so a refused command sends nothing at all. @@ -189,39 +191,12 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: with TimingContext("authentication_check"): try: auth_manager.guard() - - # Log credential source and project ID for transparency - if not config.get("output.quiet", False): - source = auth_manager.get_credential_source() - project_id = auth_manager.get_project_id() - - # Only log if not using a profile (i.e., using env vars or flags) - if source in [ - "explicit flags", - "environment variables", - ]: - # Diagnostics, not the result. print_info / - # print_warning still write to stdout outside - # agentic mode, so with an explicit `-o json` - # these three lines landed in front of the - # payload and broke `| jq` (#104, success - # path). Route them straight to stderr; the - # prefixes mirror output.py so the rendering - # is unchanged in both modes. - info = "INFO:" if _agentic else "[blue]ℹ[/blue]" - stderr_console.print( - f"{info} Using credentials from {source}" - ) - if project_id: - stderr_console.print( - f"{info} Affecting project: {project_id}" - ) - else: - warn = "WARN:" if _agentic else "[yellow]⚠[/yellow]" - stderr_console.print( - f"{warn} No project ID specified" - ) - + self._print_credential_source(config, auth_manager) + except TransportError as e: + # auth_required: the coded result on stdout and two + # stderr lines, like any other coded error. + self._report_transport_error(e, ctx, config, auth_manager) + raise SystemExit(1) from None except Exception as auth_error: # guard() already wrote the human-readable diagnosis to # stderr, so don't duplicate it -- but stdout must still @@ -310,6 +285,37 @@ def execute(self, ctx: click.Context, **kwargs: Any) -> None: if exit_code: raise SystemExit(exit_code) + def _make_client(self, config: Config, auth_manager: AuthManager) -> Any: + """The Deepgram client handed to ``handle``.""" + return DeepgramClient(config, auth_manager) + + def _print_credential_source( + self, config: Config, auth_manager: AuthManager + ) -> None: + """Say where the credentials came from when no profile supplied them. + + Diagnostics, not the result: straight to stderr, so `-o json` + output stays parseable (#104). The source is the resolver's, so a + key from DEEPGRAM_API_KEY is reported as the environment's even + though Config merges it into the profile. + "No project ID specified" only for commands that need one. + """ + if config.get("output.quiet", False): + return + source = auth_manager.get_credential_source() + if source not in ("explicit flags", "environment variables"): + return + project_id = auth_manager.get_project_id() + info = "INFO:" if _agentic else "[blue]ℹ[/blue]" + stderr_console.print(f"{info} Using credentials from {source}") + if project_id: + from rich.markup import escape + + stderr_console.print(f"{info} Affecting project: {escape(project_id)}") + elif self.requires_project: + warn = "WARN:" if _agentic else "[yellow]⚠[/yellow]" + stderr_console.print(f"{warn} No project ID specified") + def _run_handler( self, config: Config, diff --git a/packages/deepctl-core/src/deepctl_core/base_group_command.py b/packages/deepctl-core/src/deepctl_core/base_group_command.py index 1c64e565..768d1981 100644 --- a/packages/deepctl-core/src/deepctl_core/base_group_command.py +++ b/packages/deepctl-core/src/deepctl_core/base_group_command.py @@ -1,6 +1,6 @@ """Base class for group commands that can contain subcommands.""" -from typing import Any +from typing import Any, cast import click @@ -16,6 +16,34 @@ console = stderr_console +class _LazyClient: + """A :class:`DeepgramClient` built on first use. + + A group's callback runs before its subcommand parses ``--help``, so + ``dg debug --help`` must not load the SDK (and with it httpx and + ssl) just to hand the group a client it rarely uses. + """ + + __slots__ = ("_args", "_client") + + def __init__(self, config: Config, auth_manager: AuthManager) -> None: + object.__setattr__(self, "_args", (config, auth_manager)) + object.__setattr__(self, "_client", None) + + def _real(self) -> DeepgramClient: + client = object.__getattribute__(self, "_client") + if client is None: + client = DeepgramClient(*object.__getattribute__(self, "_args")) + object.__setattr__(self, "_client", client) + return cast("DeepgramClient", client) + + def __getattr__(self, name: str) -> Any: + return getattr(self._real(), name) + + def __setattr__(self, name: str, value: Any) -> None: + setattr(self._real(), name, value) + + class BaseGroupCommand(BaseCommand): """Base class for commands that act as groups containing subcommands. @@ -44,6 +72,10 @@ def __init__(self) -> None: if not hasattr(self, "invoke_without_command"): self.invoke_without_command = False + def _make_client(self, config: Config, auth_manager: AuthManager) -> Any: + """A client built only if the group's handler uses it.""" + return _LazyClient(config, auth_manager) + def add_subcommand(self, name: str, command_class: type[BaseCommand]) -> None: """Add a subcommand to this group. diff --git a/packages/deepctl-core/src/deepctl_core/client.py b/packages/deepctl-core/src/deepctl_core/client.py index ba86112e..70515bbe 100644 --- a/packages/deepctl-core/src/deepctl_core/client.py +++ b/packages/deepctl-core/src/deepctl_core/client.py @@ -7,8 +7,7 @@ from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlencode -from . import transport -from .hosts import DEFAULT_BASE_URL +from . import hosts, transport from .output import stderr_console if TYPE_CHECKING: @@ -16,8 +15,7 @@ from datetime import datetime # Bound at run time by _sdk() below, not by these imports. - from deepgram import DeepgramClient as DGClient # noqa: TC004 - from deepgram import DeepgramClientEnvironment # noqa: TC004 + from deepgram import DeepgramClient as DGClient from deepgram.core.api_error import ApiError # noqa: TC004 from .auth import AuthManager @@ -90,8 +88,9 @@ def __init__(self, config: Config, auth_manager: AuthManager): self.auth_manager = auth_manager self._client: DGClient | None = None self._project_id: str | None = None - # The REST base the SDK client was given (for redirect errors). - self._rest_base: str = DEFAULT_BASE_URL + self.transport = transport.Transport( + config, check_host=auth_manager.check_base_url + ) # Kept out of error text. self._secrets: tuple[str, ...] = () @@ -122,78 +121,26 @@ def _create_client(self) -> DGClient: if not api_key: raise ApiError(body="No API key available") - # The SDK sends the key to whatever environment it is given, so the - # base URL goes through the same host check as every other path. - base_url = self.auth_manager.check_base_url() - try: - # One connection pool per process; requests reuse its connection. - kwargs: dict[str, Any] = { - "api_key": api_key, - "httpx_client": transport.shared_client(), - } - - if base_url and base_url != DEFAULT_BASE_URL: - # Derive both the REST and WebSocket endpoints from the - # configured base URL, so a custom base works for batch REST - # *and* streaming. Preserve the transport implied by the input - # scheme (http→ws, https→wss) so a plaintext local/staging - # endpoint isn't forced onto TLS. agent_rest is required by the - # SDK environment. - rest_scheme, ws_scheme, host = _split_base_url(base_url) - self._rest_base = f"{rest_scheme}://{host}" - kwargs["environment"] = DeepgramClientEnvironment( - base=f"{rest_scheme}://{host}", - production=f"{ws_scheme}://{host}", - agent=f"{ws_scheme}://{host}", - agent_rest=f"{rest_scheme}://{host}", - ) - - # A redirect would take the request body (audio, text) to - # another host, and with the SDK's default the key too on a - # same-origin hop. Refuse it; _failed() words the error. - kwargs["follow_redirects"] = False - - client = DGClient(**kwargs) + # The Transport checks the host before the SDK gets the key, and + # derives the REST and WebSocket endpoints from the same base. + client = cast("DGClient", self.transport.sdk_client(api_key)) self._project_id = project_id self._secrets = (api_key,) return client + except hosts.CredentialHostError: + raise except Exception as e: console.print(f"[red]Error creating Deepgram client:[/red] {e}") raise ApiError(body=f"Failed to create client: {e}") - def _failed(self, what: str, e: Exception) -> Exception: - """The error to raise for a failed SDK call. - - A refused redirect passes through unchanged, and a 3xx answer - (the SDK client doesn't follow redirects) becomes the same refusal. - Anything else is wrapped as before. - """ - from .websocket import RedirectRefused, redirect_refused - - if isinstance(e, RedirectRefused): - return e - status = getattr(e, "status_code", None) - if isinstance(e, ApiError) and status is not None and 300 <= status < 400: - headers = {k.lower(): v for k, v in (e.headers or {}).items()} - return redirect_refused( - self._rest_base, headers.get("location"), command="dg", payload=None - ) - return ApiError(body=f"{what}: {e}") - def _mapped(self, chunks: Iterator[bytes]) -> Iterator[bytes]: - """Yield ``chunks``; a request failure while reading them is mapped, - and a 3xx answer becomes the redirect refusal.""" - from .websocket import RedirectRefused - + """Yield ``chunks``; a request failure while reading them is mapped.""" try: yield from chunks except Exception as e: transport.raise_for(e, self._secrets) - refused = self._failed("", e) - if isinstance(refused, RedirectRefused): - raise refused from e raise # ── Speech-to-Text (pre-recorded) ────────────────────────────── @@ -246,7 +193,7 @@ def transcribe_file( raise except Exception as e: transport.raise_for(e, self._secrets, timeout_s=timeout_s) - raise self._failed("Transcription failed", e) + raise ApiError(body=f"Transcription failed: {e}") def transcribe_url( self, @@ -281,7 +228,7 @@ def transcribe_url( except Exception as e: transport.raise_for(e, self._secrets, timeout_s=timeout_s) - raise self._failed("Transcription failed", e) + raise ApiError(body=f"Transcription failed: {e}") # ── Text-to-Speech ───────────────────────────────────────────── @@ -311,7 +258,7 @@ def speak_text( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text-to-speech failed", e) + raise ApiError(body=f"Text-to-speech failed: {e}") def speak_text_stream( self, @@ -353,7 +300,7 @@ def speak_text_stream( # its connect sent them; creating the client runs the key and host # checks. headers = dict(self.client._client_wrapper.get_headers()) - url = f"{self.ws_base_url()}/v2/speak?{urlencode(query)}" + url = f"{self.transport.ws_base().rstrip('/')}/v2/speak?{urlencode(query)}" try: with connect_sync( url, @@ -372,15 +319,7 @@ def speak_text_stream( yield message except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text-to-speech streaming failed", e) - - def ws_base_url(self) -> str: - """The WebSocket base for the checked base URL, without a slash.""" - base_url = self.auth_manager.check_base_url() - if not base_url or base_url == DEFAULT_BASE_URL: - base_url = DEFAULT_BASE_URL - _rest, ws_scheme, host = _split_base_url(str(base_url)) - return f"{ws_scheme}://{host}" + raise ApiError(body=f"Text-to-speech streaming failed: {e}") # ── Text Intelligence (Read) ─────────────────────────────────── @@ -415,7 +354,7 @@ def analyze_text( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Text analysis failed", e) + raise ApiError(body=f"Text analysis failed: {e}") # ── Models ───────────────────────────────────────────────────── @@ -431,7 +370,7 @@ def list_models(self, include_outdated: bool = False) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list models", e) + raise ApiError(body=f"Failed to list models: {e}") def get_model(self, model_id: str) -> dict[str, Any]: """Get a specific model by ID.""" @@ -441,7 +380,7 @@ def get_model(self, model_id: str) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get model", e) + raise ApiError(body=f"Failed to get model: {e}") # ── Projects ─────────────────────────────────────────────────── @@ -453,7 +392,7 @@ def get_projects(self) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get projects", e) + raise ApiError(body=f"Failed to get projects: {e}") def get_project(self, project_id: str | None = None) -> dict[str, Any]: """Get specific project.""" @@ -464,7 +403,7 @@ def get_project(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get project", e) + raise ApiError(body=f"Failed to get project: {e}") def create_project(self, name: str, company: str | None = None) -> dict[str, Any]: """Create a new project.""" @@ -478,7 +417,7 @@ def create_project(self, name: str, company: str | None = None) -> dict[str, Any except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create project", e) + raise ApiError(body=f"Failed to create project: {e}") # ── Usage ────────────────────────────────────────────────────── @@ -504,7 +443,7 @@ def get_usage( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get usage", e) + raise ApiError(body=f"Failed to get usage: {e}") # ── API Keys ─────────────────────────────────────────────────── @@ -525,7 +464,7 @@ def list_keys( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list keys", e) + raise ApiError(body=f"Failed to list keys: {e}") def create_key( self, @@ -557,7 +496,7 @@ def create_key( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create key", e) + raise ApiError(body=f"Failed to create key: {e}") def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: """Get a specific API key.""" @@ -568,7 +507,7 @@ def get_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get key", e) + raise ApiError(body=f"Failed to get key: {e}") def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, Any]: """Delete an API key.""" @@ -579,7 +518,7 @@ def delete_key(self, key_id: str, project_id: str | None = None) -> dict[str, An except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to delete key", e) + raise ApiError(body=f"Failed to delete key: {e}") # ── Requests ─────────────────────────────────────────────────── @@ -618,7 +557,7 @@ def list_requests( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list requests", e) + raise ApiError(body=f"Failed to list requests: {e}") def get_request( self, request_id: str, project_id: str | None = None @@ -631,7 +570,7 @@ def get_request( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get request", e) + raise ApiError(body=f"Failed to get request: {e}") # ── Billing ──────────────────────────────────────────────────── @@ -660,7 +599,7 @@ def get_billing_breakdown( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get billing breakdown", e) + raise ApiError(body=f"Failed to get billing breakdown: {e}") def get_balances(self, project_id: str | None = None) -> dict[str, Any]: """Get project balances.""" @@ -671,7 +610,7 @@ def get_balances(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to get balances", e) + raise ApiError(body=f"Failed to get balances: {e}") # ── Members ──────────────────────────────────────────────────── @@ -684,7 +623,7 @@ def list_members(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list members", e) + raise ApiError(body=f"Failed to list members: {e}") def remove_member( self, member_id: str, project_id: str | None = None @@ -697,7 +636,7 @@ def remove_member( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to remove member", e) + raise ApiError(body=f"Failed to remove member: {e}") def list_member_scopes( self, member_id: str, project_id: str | None = None @@ -712,7 +651,7 @@ def list_member_scopes( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list member scopes", e) + raise ApiError(body=f"Failed to list member scopes: {e}") def update_member_scopes( self, member_id: str, scope: str, project_id: str | None = None @@ -727,7 +666,7 @@ def update_member_scopes( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to update member scopes", e) + raise ApiError(body=f"Failed to update member scopes: {e}") # ── Invites ──────────────────────────────────────────────────── @@ -740,7 +679,7 @@ def list_invites(self, project_id: str | None = None) -> dict[str, Any]: except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to list invites", e) + raise ApiError(body=f"Failed to list invites: {e}") def create_invite( self, email: str, scope: str, project_id: str | None = None @@ -755,7 +694,7 @@ def create_invite( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to create invite", e) + raise ApiError(body=f"Failed to create invite: {e}") def delete_invite( self, email: str, project_id: str | None = None @@ -768,7 +707,7 @@ def delete_invite( except Exception as e: transport.raise_for(e, self._secrets) - raise self._failed("Failed to delete invite", e) + raise ApiError(body=f"Failed to delete invite: {e}") # ── Utilities ────────────────────────────────────────────────── diff --git a/packages/deepctl-core/src/deepctl_core/credentials.py b/packages/deepctl-core/src/deepctl_core/credentials.py new file mode 100644 index 00000000..b3bd5786 --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/credentials.py @@ -0,0 +1,541 @@ +"""One place that decides which API key and project ID a run uses. + +:func:`resolve` returns a :class:`Resolved`: the key, the project ID, where +each came from, the profile and why that profile, and whether the user +config file's key was left unused because the file isn't trusted. Every +``AuthManager`` question (``get_api_key``, ``get_project_id``, +``get_credential_source``, ``is_authenticated``, ``has_profile_credentials``) +is answered from it, so ``mcp``, ``init``, ``debug toolkit`` and ``login`` +share one precedence. + +The 0.5.0 precedence is 0.4.2's, unchanged: ``--api-key`` > the profile's +keyring entry > the profile in the config file > ``DEEPGRAM_API_KEY``. The +config file layer already holds ``DEEPGRAM_API_KEY`` when it is set +(``Config`` merges the environment into the profile), so a file key equal to +the environment value is reported as the environment's. The 1.0 order +comes later. + +Under a hard AI-agent signal, ``Config`` drops the key of a user file that +others can write or that someone else owns; +``Config.api_key_untrusted`` says so and the result carries +``untrusted_source``. Such a file's ``dg_token_expires_at`` never deletes +the keyring entry. + +The keyring is imported only when a key is needed and no ``--api-key`` was +given. Off a terminal (and so under MCP), a read gets +:data:`KEYRING_BOUND_S` seconds in a daemon thread; when it doesn't answer, +the run goes on without it, says so once (``W-KEYRING-TIMEOUT``) and doesn't +ask again. + +A value that is still an unexpanded ``${...}`` placeholder counts as unset +(``W-ENV-PLACEHOLDER``, once per variable). + +This module imports only the standard library at import time. +""" + +from __future__ import annotations + +import os +import sys +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .config import Config + +# ``key_source`` values: `dg whoami`'s words, plus the flag. +KEY_SOURCE_FLAG = "--api-key" +KEY_SOURCE_KEYRING = "system keyring" +KEY_SOURCE_FILE = "config file" +KEY_SOURCE_ENV = "DEEPGRAM_API_KEY (env)" +KEY_SOURCE_NONE = "not set" + +# ``profile_source`` values (DX-S15). +PROFILE_SOURCE_NONE = "none" +PROFILE_SOURCE_FLAG = "--profile" +PROFILE_SOURCE_ENV = "DEEPGRAM_PROFILE" +PROFILE_SOURCE_FILE = "user-file default" + +# Keyring reads off a terminal get this long before the run goes on without +# the keyring. +KEYRING_BOUND_S = 2.0 + +# The env-vs-stored mismatch warning. Off in 0.5.0: login doesn't +# write fingerprints yet. A later release turns it on. +WARN_ENV_MISMATCH = False + +# The environment variables a placeholder can't stand in for. +PLACEHOLDER_VARIABLES = ( + "DEEPGRAM_API_KEY", + "DEEPGRAM_PROFILE", + "DEEPGRAM_PROJECT_ID", + "DEEPGRAM_BASE_URL", +) + +# Catalogue text (error_codes.json "strings"). Filled in with +# str.replace: the text itself holds braces. +W_ENV_PLACEHOLDER = ( + "{prog} ignored {variable}: it holds a placeholder that was never filled in (${…})." +) +W_KEYRING_TIMEOUT = ( + "The system keyring didn't answer within 2 s, so {prog} didn't use the key " + "stored there." +) + +_EXPIRY_INVALID = "Your Deepgram credential expiry is invalid. Run `dg login` again." +_EXPIRY_PAST = "Your Deepgram credential has expired. Run `dg login` again." + + +@dataclass(frozen=True) +class Resolved: + """The credentials one run uses, and where they came from.""" + + api_key: str | None + project_id: str | None + key_source: str + profile: str + profile_source: str + # The user config file whose key wasn't used (display path), or None. + untrusted_source: str | None = None + # Why a stored credential was unusable when nothing else was set. + expiry_error: str | None = None + # 0.4.2's has_profile_credentials(): a usable keyring entry, or a key in + # the profile as Config holds it (which includes DEEPGRAM_API_KEY). + profile_has_key: bool = False + profile_has_project: bool = False + project_source: str = "none" # "flag", "profile", "env" or "none" + # DEEPGRAM_API_KEY is set and differs from the stored key that won. + # Always False while WARN_ENV_MISMATCH is off. + env_mismatch: bool = False + # The keyring entry an expired credential in a trusted file leaves behind + # (``api-key.``); removed only by a resolution with prune=True. + prune_target: str | None = None + + +# ── Placeholders ───────────────────────────────────────────────────── + +_lock = threading.Lock() +# variable -> the placeholder text it held (to recognise it inside Config). +_dropped: dict[str, str] = {} +_warned: set[str] = set() + + +def is_placeholder(value: object) -> bool: + """True for a value that still holds an unexpanded ``${...}``.""" + return isinstance(value, str) and "${" in value + + +def drop_env_placeholders() -> list[str]: + """Remove placeholder values of :data:`PLACEHOLDER_VARIABLES` from the + environment, so ``Config`` and everything after it see them as unset. + + Returns the variables removed by this call. Safe to call more than once. + """ + removed = [] + with _lock: + for name in PLACEHOLDER_VARIABLES: + value = os.environ.get(name) + if is_placeholder(value): + _dropped[name] = str(value) + del os.environ[name] + removed.append(name) + return removed + + +def env_value(name: str) -> str | None: + """``os.environ[name]``, with an empty value or a placeholder treated as + unset.""" + return _env(name) + + +def _env(name: str) -> str | None: + """``os.environ[name]``, with a placeholder treated as unset.""" + value = os.environ.get(name) + if is_placeholder(value): + with _lock: + _dropped.setdefault(name, str(value)) + return None + return value or None + + +def _was_placeholder(name: str, value: object) -> bool: + """True when ``value`` is the placeholder ``name`` holds or held (merged + by a Config built before :func:`drop_env_placeholders` ran).""" + if not is_placeholder(value): + return False + if os.environ.get(name) == value: + with _lock: + _dropped.setdefault(name, str(value)) + return True + with _lock: + return _dropped.get(name) == value + + +def flush_warnings() -> None: + """Print ``W-ENV-PLACEHOLDER`` once per ignored variable.""" + with _lock: + names = [n for n in PLACEHOLDER_VARIABLES if n in _dropped and n not in _warned] + _warned.update(names) + for name in names: + _warn(W_ENV_PLACEHOLDER.replace("{variable}", name)) + + +def _warn(text: str) -> None: + from rich.markup import escape + + from . import output, transport + + message = text.replace("{prog}", transport.prog() or "dg") + output.print_warning(escape(message), stderr=True, ignore_quiet=True) + + +# ── Keyring ────────────────────────────────────────────────────────── + +_keyring_timed_out = False + + +def _interactive() -> bool: + """A person at a terminal (where a keyring may ask to be unlocked).""" + try: + return bool(sys.stdin and sys.stdin.isatty() and sys.stderr.isatty()) + except (AttributeError, ValueError, OSError): + return False + + +def keyring_timed_out() -> bool: + """True once a keyring read in this process didn't answer in time.""" + return _keyring_timed_out + + +def _keyring_call(name: str, *args: str) -> Any: + """``keyring.(*args)``, bounded off a terminal. + + Raises what the keyring raises. Returns ``None`` without calling it once + a read has timed out in this process. + """ + global _keyring_timed_out + if _keyring_timed_out: + return None + from . import auth + + def call() -> Any: + # Through deepctl_core.auth, so tests that patch auth.keyring apply; + # the import itself runs here too, inside the bound. + module = vars(auth).get("keyring") or auth._bind_module("keyring") + return getattr(module, name)(*args) + + if _interactive(): + return call() + box: dict[str, Any] = {} + + def work() -> None: + try: + box["value"] = call() + except BaseException as exc: # handed to the caller below + box["error"] = exc + + thread = threading.Thread(target=work, name="deepctl-keyring", daemon=True) + thread.start() + thread.join(KEYRING_BOUND_S) + if thread.is_alive(): + with _lock: + first = not _keyring_timed_out + _keyring_timed_out = True + if first: + _warn(W_KEYRING_TIMEOUT) + return None + if "error" in box: + raise box["error"] + return box.get("value") + + +class KeyringTimeout(Exception): + """The keyring didn't answer within :data:`KEYRING_BOUND_S` (this read or + an earlier one in this process).""" + + +def keyring_read(username: str) -> str | None: + """The keyring's value for ``username``, bounded like every resolver read. + + Raises what the keyring raises, and :class:`KeyringTimeout` when it + doesn't answer in time, so callers that fall back on an error keep doing + so. Returns ``None`` when there's no entry. + """ + from .auth import KEYRING_SERVICE + + if _keyring_timed_out: + raise KeyringTimeout() + value = _keyring_call("get_password", KEYRING_SERVICE, username) + if _keyring_timed_out: + raise KeyringTimeout() + return value if isinstance(value, str) and value else None + + +def keyring_get(username: str) -> str | None: + """The keyring's value for ``username``; ``None`` if absent, failing or + timed out.""" + try: + return keyring_read(username) + except Exception: + return None + + +def keyring_delete(username: str) -> None: + """Remove a keyring entry when present, without masking keyring failures.""" + from .auth import KEYRING_SERVICE, console + + try: + _keyring_call("delete_password", KEYRING_SERVICE, username) + except Exception as exc: + from keyring.errors import PasswordDeleteError + + if isinstance(exc, PasswordDeleteError): + return + console.print( + f"[yellow]Warning:[/yellow] Could not remove legacy credential: {exc}" + ) + + +# ── Resolution ─────────────────────────────────────────────────────── + + +def credential_expiry(config: Config, profile_name: str) -> tuple[bool, str | None]: + """Whether the profile's finite credential is unusable, and why.""" + expires_at = config.get_profile(profile_name).dg_token_expires_at + if not isinstance(expires_at, str): + return False, None + try: + parsed = datetime.fromisoformat(expires_at) + if parsed.tzinfo is None: + raise ValueError + except (TypeError, ValueError): + return True, _EXPIRY_INVALID + if parsed <= datetime.now(timezone.utc): + return True, _EXPIRY_PAST + return False, None + + +def _user_value(config: Config, *path: str) -> Any: + data: Any = getattr(config, "_user_data", None) + for part in path: + if not isinstance(data, dict): + return None + data = data.get(part) + return data + + +def profile_name(config: Config) -> tuple[str, str]: + """The profile this run uses and why (``profile_source``).""" + name = config.profile or "default" + explicit = getattr(config, "_explicit_profile", None) + env_profile = os.environ.get("DEEPGRAM_PROFILE") + if explicit and name == explicit: + return name, PROFILE_SOURCE_FLAG + if is_placeholder(name) and _was_placeholder("DEEPGRAM_PROFILE", name): + # Config took the placeholder as the profile name: use the one the + # file names instead. + name = ( + _user_value(config, "active_profile") + or _user_value(config, "default_profile") + or "default" + ) + name = str(name) + source = PROFILE_SOURCE_FILE if name != "default" else PROFILE_SOURCE_NONE + return name, source + if config.get("active_profile") == name: + return name, PROFILE_SOURCE_FILE + if env_profile and name == env_profile: + return name, PROFILE_SOURCE_ENV + if _user_value(config, "default_profile") == name: + return name, PROFILE_SOURCE_FILE + return name, PROFILE_SOURCE_NONE + + +def _profile_key(config: Config, name: str) -> str | None: + """The profile's key as Config holds it (file, or the merged env value).""" + key = config.get_profile(name).api_key + if not key: + return None + if is_placeholder(key): + if not _was_placeholder("DEEPGRAM_API_KEY", key): + return None + # Config merged the placeholder over the file's key: use the file's. + if getattr(config, "api_key_untrusted", False) is True: + return None + stored = _user_value(config, "profiles", name, "api_key") + return ( + stored if isinstance(stored, str) and not is_placeholder(stored) else None + ) + return key + + +def _profile_project(config: Config, name: str) -> str | None: + project = config.get_profile(name).project_id + if not project: + return None + if is_placeholder(project): + if not _was_placeholder("DEEPGRAM_PROJECT_ID", project): + return None + stored = _user_value(config, "profiles", name, "project_id") + return ( + stored if isinstance(stored, str) and not is_placeholder(stored) else None + ) + return project + + +def resolve( + ctx: Any, + *, + profile: str | None = None, + ignore_env: bool = False, + need_key: bool = True, + prune: bool = True, +) -> Resolved: + """The credentials for ``ctx``: anything with ``config``, + ``explicit_api_key`` and ``explicit_project_id`` (an ``AuthManager``). + + ``profile`` asks about another profile than the run's. ``ignore_env`` + leaves ``DEEPGRAM_API_KEY`` and ``DEEPGRAM_PROJECT_ID`` out. + ``need_key=False`` answers project questions without reading the keyring. + ``prune=False`` keeps the answer read-only: an expired credential's + keyring entry stays (whoami and the other introspection, as in 0.4.2); + commands that use the key prune it, as 0.4.2's ``get_api_key`` did. + Raises nothing; ``expiry_error`` says why a stored credential wasn't used. + """ + try: + resolved = _resolve(ctx, profile, ignore_env, need_key) + if prune: + prune_expired(resolved) + return resolved + finally: + flush_warnings() + + +def prune_expired(resolved: Resolved) -> None: + """Remove the keyring entry of an expired credential (trusted file only).""" + if resolved.prune_target: + keyring_delete(resolved.prune_target) + + +def _resolve( + ctx: Any, profile: str | None, ignore_env: bool, need_key: bool +) -> Resolved: + config: Config = ctx.config + if profile: + name, profile_source = profile, PROFILE_SOURCE_FLAG + else: + name, profile_source = profile_name(config) + untrusted = getattr(config, "api_key_untrusted", False) is True + untrusted_source = None + if untrusted: + from . import messages + + untrusted_source = messages.display_path(config.config_path) + + env_key = None if ignore_env else _env("DEEPGRAM_API_KEY") + env_project = None if ignore_env else _env("DEEPGRAM_PROJECT_ID") + + # Project ID: flag > profile (as Config holds it) > env. + project_file = _profile_project(config, name) + raw_env_project = _env("DEEPGRAM_PROJECT_ID") + if ctx.explicit_project_id: + project_id, project_source = ctx.explicit_project_id, "flag" + elif project_file: + project_id = project_file + project_source = ( + "env" if raw_env_project and project_file == raw_env_project else "profile" + ) + elif env_project: + project_id, project_source = env_project, "env" + else: + project_id, project_source = None, "none" + + def result( + key: str | None, + source: str, + *, + has_key: bool = False, + expiry_error: str | None = None, + env_mismatch: bool = False, + ) -> Resolved: + # Only a trusted file's expiry may remove the keyring entry. + prune_target = f"api-key.{name}" if expired and not untrusted else None + return Resolved( + api_key=key, + project_id=project_id, + key_source=source, + profile=name, + profile_source=profile_source, + untrusted_source=untrusted_source, + expiry_error=expiry_error, + profile_has_key=has_key, + profile_has_project=bool(project_file), + project_source=project_source, + env_mismatch=env_mismatch, + prune_target=prune_target, + ) + + expired = False + if ctx.explicit_api_key: + return result(ctx.explicit_api_key, KEY_SOURCE_FLAG) + if not need_key: + return result(None, KEY_SOURCE_NONE) + + file_key = _profile_key(config, name) + raw_env_key = _env("DEEPGRAM_API_KEY") + expired, expiry_error = credential_expiry(config, name) + if not expired: + stored = keyring_get(f"api-key.{name}") + if stored: + mismatch = _mismatch(stored, env_key) + return result( + stored, KEY_SOURCE_KEYRING, has_key=True, env_mismatch=mismatch + ) + if file_key: + source = ( + KEY_SOURCE_ENV + if raw_env_key and file_key == raw_env_key + else KEY_SOURCE_FILE + ) + # As in 0.4.2, ignore_env doesn't skip the env value Config + # merged into the profile; it skips only the last fallback. + mismatch = source == KEY_SOURCE_FILE and _mismatch(file_key, env_key) + return result(file_key, source, has_key=True, env_mismatch=mismatch) + + if env_key: + return result(env_key, KEY_SOURCE_ENV, expiry_error=expiry_error) + return result(None, KEY_SOURCE_NONE, expiry_error=expiry_error) + + +# ── Env-vs-stored mismatch ─────────────────────────────────────────── + + +def fingerprint(key: str) -> str: + """A short, non-reversible fingerprint of a key, for comparisons.""" + import hashlib + + return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + + +def _mismatch(stored: str, env_key: str | None) -> bool: + """True when DEEPGRAM_API_KEY is set and isn't the stored key that won. + + Compares fingerprints, never keys, so a later release can compare with the + fingerprint login will store. Behind :data:`WARN_ENV_MISMATCH`, which is + off in 0.5.0: login doesn't write fingerprints yet, and a later release + adds the warning text and turns it on. + """ + if not WARN_ENV_MISMATCH or not env_key: + return False + return fingerprint(stored) != fingerprint(env_key) + + +def _reset_for_tests() -> None: + """Forget the per-process state (placeholders, warnings, keyring timeout).""" + global _keyring_timed_out + with _lock: + _dropped.clear() + _warned.clear() + _keyring_timed_out = False diff --git a/packages/deepctl-core/src/deepctl_core/messages.py b/packages/deepctl-core/src/deepctl_core/messages.py index 870b51c6..10639f8c 100644 --- a/packages/deepctl-core/src/deepctl_core/messages.py +++ b/packages/deepctl-core/src/deepctl_core/messages.py @@ -31,7 +31,7 @@ # The command comes last, with no trailing period, so it can be copied. E_FH_PROFILE_HINT = ( "If you trust {host}, add it to api.allowed_hosts in {user_config}. To " - "switch back to Deepgram, run: dg --base-url https://api.deepgram.com " + "switch back to Deepgram, run: dg --base-url https://api.deepgram.com " # host-literal: user text "login --profile {profile} --save-base-url --force-write" ) # Agent-mode hint shared by every foreign_host error. Never names a flag, diff --git a/packages/deepctl-core/src/deepctl_core/transport.py b/packages/deepctl-core/src/deepctl_core/transport.py index bb1030c5..6b4293f8 100644 --- a/packages/deepctl-core/src/deepctl_core/transport.py +++ b/packages/deepctl-core/src/deepctl_core/transport.py @@ -1,19 +1,27 @@ -"""One HTTP client per process, and the map from transport failures to codes. - -Every REST call the SDK facade makes goes through :func:`shared_client`: one -``httpx.Client`` (one connection pool) per process, created on first use so -``--version`` and ``--help`` never import httpx. The client carries no -credentials and no base URL. The key and the host still come from -``DeepgramClient``, which takes the base URL from -:func:`deepctl_core.hosts.check_credential_host` before any request carries -the key; this module never decides where a credential may go. +"""Where API requests go, what they carry, and what their failures mean. + +:class:`Transport` owns the REST and WebSocket base URLs, which come only +from :func:`deepctl_core.hosts.check_credential_host` (``http`` -> ``ws``, +``https`` -> ``wss``), and attaches ``Authorization`` only after that check +has passed for the exact origin. Requests go through :func:`shared_client`: +one ``httpx.Client`` (one connection pool) per process, created on first use +so ``--version`` and ``--help`` never import httpx. It follows no redirects +(a 3xx with credentials is an error, never a second request), keeps no +cookies, and verifies TLS. A person's ``DEEPGRAM_CLI_INSECURE=1`` (or +``true``) skips verification for localhost, 127.0.0.1 and [::1] only, and +never under an AI agent (``auth._insecure_allowed`` and +``auth._insecure_mounts``, as in 0.4.2). :func:`verified_client` is the +always-verified client for ``dg api``; WebSocket streams always verify. +The SDK facade builds its client through :attr:`Transport.backend`, a seam +for other backends (#142's SageMaker), with the REST backend as the default. :func:`raise_for` turns a failed request into a :class:`TransportError` with a catalogue code (``transport_mapping`` in ``error_codes.json``, in -order): DNS, connect, TLS, reset or a truncated 2xx body -> +order): DNS, connect, TLS, reset, a truncated 2xx body or a 3xx -> ``network_unavailable``; timeout -> ``timeout``; 401 -> ``auth_invalid``; 403 -> ``auth_forbidden``; 404 -> ``not_found``; 429 -> ``rate_limited``; 400, 422 and any other 4xx -> ``invalid_argument``; 5xx -> ``server_error``. +A WebSocket upgrade refused with an HTTP status maps through the same table. ``BaseCommand.execute`` renders it: a coded result on stdout under a machine format, and two stderr lines. """ @@ -28,16 +36,25 @@ import threading from typing import TYPE_CHECKING, Any +from . import credentials + if TYPE_CHECKING: + from collections.abc import Callable + import httpx from .auth import AuthManager + from .config import Config # The SDK's own default for a client it builds itself. DEFAULT_TIMEOUT_S = 60.0 _lock = threading.Lock() _client: httpx.Client | None = None +# True when _client has DEEPGRAM_CLI_INSECURE's unverified loopback mounts. +_client_insecure = False +# The always-verified client for dg api, when _client isn't one. +_verified: httpx.Client | None = None # Where the most recent request went, for the error text: method, host and # path only (never the query string, which can carry values from argv). _last_request: dict[str, str] = {} @@ -46,12 +63,18 @@ def _record_request(request: httpx.Request) -> None: _last_request.clear() _last_request.update( - method=request.method, host=request.url.host, path=request.url.path + method=request.method, + host=request.url.host, + path=request.url.path, + origin=f"{request.url.scheme}://{request.url.netloc.decode('ascii')}", ) def _install(client: httpx.Client) -> httpx.Client: _no_cookies(client) + # A redirect could take the key, or a request body, to another origin + # a 3xx is an error (see map_exception), never followed. + client.follow_redirects = False hooks = client.event_hooks hooks.setdefault("request", []) if _record_request not in hooks["request"]: @@ -60,23 +83,58 @@ def _install(client: httpx.Client) -> httpx.Client: return client +def _new_client(*, insecure_loopback: bool) -> httpx.Client: + import httpx + + from .auth import _insecure_mounts, _warn_if_verified + + if insecure_loopback: + # As AuthManager.client (0.4.2): verification stays on, and only the + # loopback hosts get unverified transports. + return httpx.Client( + timeout=DEFAULT_TIMEOUT_S, + follow_redirects=False, + verify=True, + mounts=_insecure_mounts(), + event_hooks={"request": [_warn_if_verified]}, + ) + return httpx.Client(timeout=DEFAULT_TIMEOUT_S, follow_redirects=False) + + def shared_client() -> httpx.Client: """The process's one ``httpx.Client``, created on first use. - It follows no redirects: a 3xx could take the key, or a request body, - to another host (see ``DeepgramClient._failed``). + It follows no redirects, keeps no cookies and verifies TLS. A person's + ``DEEPGRAM_CLI_INSECURE=1`` (or ``true``) skips verification for + localhost, 127.0.0.1 and [::1] only; never under an AI agent + (``auth._insecure_allowed``, ``auth._insecure_mounts``). """ - global _client + global _client, _client_insecure with _lock: if _client is None: - import httpx + from .auth import _insecure_allowed - _client = _install( - httpx.Client(timeout=DEFAULT_TIMEOUT_S, follow_redirects=False) - ) + _client_insecure = _insecure_allowed() + _client = _install(_new_client(insecure_loopback=_client_insecure)) return _client +def verified_client() -> httpx.Client: + """A client that verifies TLS for every host (``dg api``, as in 0.4.1). + + The shared client itself, unless ``DEEPGRAM_CLI_INSECURE`` gave it + unverified loopback transports; then a second client that has none. + """ + global _verified + client = shared_client() + with _lock: + if not _client_insecure: + return client + if _verified is None: + _verified = _install(_new_client(insecure_loopback=False)) + return _verified + + def _no_cookies(client: httpx.Client) -> None: """Keep no cookies: one process can serve several profiles and hosts.""" from http.cookiejar import DefaultCookiePolicy @@ -86,15 +144,210 @@ def _no_cookies(client: httpx.Client) -> None: def _set_client(client: httpx.Client | None) -> None: """Replace the shared client (tests pass one with an ``httpx.MockTransport``).""" - global _client + global _client, _client_insecure, _verified with _lock: _client = _install(client) if client is not None else None + _client_insecure = False + _verified = None _last_request.clear() +# ── Base URLs and credentials ──────────────────────────────────────── + + +def ws_url(base_url: str) -> str: + """The WebSocket form of a REST base URL: ``http`` -> ``ws``, + ``https`` (or none) -> ``wss``; host, port and path unchanged.""" + scheme, sep, rest = base_url.partition("://") + if not sep: + return f"wss://{base_url}" + scheme = scheme.lower() + if scheme in ("http", "ws"): + return f"ws://{rest}" + return f"wss://{rest}" + + +def rest_url(base_url: str) -> str: + """The REST form of a base URL: ``ws`` -> ``http``, ``wss`` (or none) -> + ``https``.""" + scheme, sep, rest = base_url.partition("://") + if not sep: + return f"https://{base_url}" + scheme = scheme.lower() + if scheme in ("http", "ws"): + return f"http://{rest}" + return f"https://{rest}" + + +def default_host() -> str: + """The Deepgram API host requests go to by default (``hosts.DEFAULT_BASE_URL``).""" + from urllib.parse import urlsplit + + from .hosts import DEFAULT_BASE_URL + + return urlsplit(DEFAULT_BASE_URL).hostname or DEFAULT_BASE_URL + + +# One random ID per run, sent as x-deepgram-session-id with every request +# that carries the key, so Deepgram can group a run's requests. It says +# nothing about the user or the machine. +_session_id: str | None = None + +SESSION_HEADER = "x-deepgram-session-id" + + +def session_id() -> str: + """This run's ``x-deepgram-session-id``, the same on every request.""" + import uuid + + global _session_id + if _session_id is None: + _session_id = str(uuid.uuid4()) + return _session_id + + +def request_headers(api_key: str) -> dict[str, str]: + """The headers every Deepgram API request and WebSocket open carries. + + The SDK's identification (``User-Agent: deepgram-sdk/`` and its + ``X-Fern-*`` headers, built as its ``get_headers`` builds them), this + run's session ID, and ``Authorization``. The SDK client gets the same + session ID (:class:`RestBackend`), so requests dg sends itself and + requests the SDK sends identify the run alike. Callers check the host + first; :meth:`Transport.auth_headers` does. + """ + import platform + from importlib import metadata + + headers: dict[str, str] = {} + try: + version = metadata.version("deepgram-sdk") + except metadata.PackageNotFoundError: + pass + else: + headers = { + "User-Agent": "deepgram-sdk/" + version, + "X-Fern-Language": "Python", + "X-Fern-Runtime": f"python/{platform.python_version()}", + "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", + "X-Fern-SDK-Name": "deepgram-sdk", + "X-Fern-SDK-Version": version, + } + headers[SESSION_HEADER] = session_id() + headers["Authorization"] = f"Token {api_key}" + return headers + + +class RestBackend: + """The default backend: the Deepgram SDK over the shared ``httpx.Client``. + + A backend turns a checked base URL and a key into the SDK client the + facade uses. #142's SageMaker backend plugs in here (R-16). + """ + + name = "rest" + + def sdk_client(self, api_key: str, rest_base: str, ws_base: str) -> Any: + # Through the facade module's lazily bound names (tests patch them). + from . import client as facade + from .hosts import DEFAULT_BASE_URL + + facade._sdk() + sdk: dict[str, Any] = vars(facade) + + kwargs: dict[str, Any] = { + "api_key": api_key, + # One connection pool per process; requests reuse its connection. + "httpx_client": shared_client(), + # The run's ID, so SDK requests match request_headers(). + "session_id": session_id(), + } + if rest_base != DEFAULT_BASE_URL: + # Both the REST and the WebSocket endpoints come from the checked + # base URL, so a custom base works for batch REST and streaming. + # agent_rest is required by the SDK environment. + kwargs["environment"] = sdk["DeepgramClientEnvironment"]( + base=rest_base, + production=ws_base, + agent=ws_base, + agent_rest=rest_base, + ) + return sdk["DGClient"](**kwargs) + + +class Transport: + """Where this run's API requests go and what they carry. + + Base URLs come from :func:`deepctl_core.hosts.check_credential_host` + only; it raises ``CredentialHostError`` (already reported) when the host + isn't trusted with credentials, before anything carries the key. + """ + + def __init__( + self, + config: Config, + backend: Any = None, + *, + check_host: Callable[[], str] | None = None, + ) -> None: + """``check_host`` is ``AuthManager.check_base_url`` when there is one + (the same ``check_credential_host`` call on the same config).""" + self.config = config + self.backend = backend if backend is not None else RestBackend() + self._check_host = check_host + + def rest_base(self) -> str: + """The checked REST base URL (scheme, host, port and path).""" + if self._check_host is not None: + return rest_url(self._check_host()) + from .hosts import check_credential_host + + return rest_url(check_credential_host(self.config)) + + def ws_base(self) -> str: + """The checked WebSocket base URL (``http`` -> ``ws``, ``https`` -> ``wss``).""" + return ws_url(self.rest_base()) + + def auth_headers(self, api_key: str) -> dict[str, str]: + """:func:`request_headers` for ``api_key`` (identification, session + ID and ``Authorization``), once the host check has passed.""" + self.rest_base() + return request_headers(api_key) + + @property + def client(self) -> httpx.Client: + """The process's shared ``httpx.Client``.""" + return shared_client() + + def sdk_client(self, api_key: str) -> Any: + """The SDK client for ``api_key``, built by the backend once the host + check has passed.""" + rest = self.rest_base() + return self.backend.sdk_client(api_key, rest, ws_url(rest)) + + # ── Catalogue text (error_codes.json "strings") ────────────────────── STRINGS: dict[str, str] = { + # The 0.4.x error text, frozen: scripts match on it (legacy_error_text_frozen). + "E-AUTH-REQUIRED": "DEEPGRAM_API_KEY is not set", + "H-AUTH-REQUIRED": "Run {prog} login, or set DEEPGRAM_API_KEY.", + "A-AUTH-REQUIRED": ( + "Run {prog} login and give the user the verification link and code it " + "prints. Never ask the user for an API key in a chat, and don't search " + "files for one." + ), + "C-AUTH-REQUIRED": ( + "Set DEEPGRAM_API_KEY in this job's environment, or run {prog} login." + ), + # 0.4.1's messages.E_KEY_UNTRUSTED and W_CFG_UNTRUSTED_FILE_HINT, verbatim. + "E-KEY-UNTRUSTED": ( + "dg didn't use the API key in {user_config} (see the warning above), and no " + "other API key is set." + ), + "W_CFG_UNTRUSTED_FILE_HINT": ( + "A person must fix this file. If you are an AI agent, ask the user." + ), "E-AUTH-INVALID": "Deepgram rejected the API key (HTTP 401).", "H-AUTH-INVALID": ( "The API key may have expired or been revoked. Run {prog} login to log " @@ -141,6 +394,9 @@ def _set_client(client: httpx.Client | None) -> None: "Retry once. If it fails again, tell the user the connection to Deepgram " "keeps dropping." ), + "E-NETWORK-DROPPED": ( + "The connection to {host} dropped before the response was complete ({reason})." + ), "E-NETWORK-UNREADABLE": "Got a response from {host} that {prog} couldn't read.", "H-NETWORK-UNREADABLE": ( "A proxy, captive portal or firewall may be answering in Deepgram's " @@ -151,6 +407,20 @@ def _set_client(client: httpx.Client | None) -> None: "network (a proxy, captive portal or firewall) is answering in " "Deepgram's place." ), + # 0.4.1's redirect refusal (origins only); {sent} is "nothing was sent + # there" for REST and "no audio was sent" (or text) for a WebSocket. + "E-NETWORK-REDIRECT": ( + "{origin} answered with a redirect to {location_origin}. {command} " + "doesn't follow redirects, so {sent}." + ), + "H-NETWORK-REDIRECT": ( + "A network sign-in page or proxy may be redirecting requests. If {host} " + "is your own server, use the address it redirects to." + ), + "A-NETWORK-REDIRECT": ( + "Tell the user {host} redirected the request. Don't retry until they " + "have checked the network or the configured URL." + ), "E-TIMEOUT": "{host} didn't respond within {timeout_s} s.", "H-TIMEOUT": "Try again. Large files take longer to upload on slow connections.", "A-TIMEOUT": ( @@ -188,11 +458,26 @@ def _set_client(client: httpx.Client | None) -> None: "Retry once after a few seconds. If it fails again, tell the user and " "include details.request_id." ), + # dg update: the package index didn't answer. + "E-UPDATE-CHECK": "Couldn't check for updates: couldn't reach {host}.", + "H-UPDATE-CHECK": ( + "Check your network connection, or update {prog} through the tool you " + "installed it with." + ), + "A-UPDATE-CHECK": ( + "The update check needs network access. Tell the user; don't retry in a loop." + ), } # code -> (error message ID, human hint, agent hint, automation hint or None) Messages = tuple[str, str, str, "str | None"] CODES: dict[str, Messages] = { + "auth_required": ( + "E-AUTH-REQUIRED", + "H-AUTH-REQUIRED", + "A-AUTH-REQUIRED", + "C-AUTH-REQUIRED", + ), "auth_invalid": ( "E-AUTH-INVALID", "H-AUTH-INVALID", @@ -216,10 +501,23 @@ def _set_client(client: httpx.Client | None) -> None: None, ), "server_error": ("E-SERVER", "H-SERVER", "A-SERVER", None), + "update_check_failed": ( + "E-UPDATE-CHECK", + "H-UPDATE-CHECK", + "A-UPDATE-CHECK", + None, + ), } -# network_unavailable's other messages: the response was cut off, or it -# arrived complete but isn't Deepgram's (a proxy or captive portal page). +# network_unavailable's other messages: the connection dropped after the +# request started, the response was cut off, it arrived complete but isn't +# Deepgram's (a proxy or captive portal page), or it was a redirect. +NETWORK_DROPPED: Messages = ( + "E-NETWORK-DROPPED", + "H-NETWORK-CUT", + "A-NETWORK-CUT", + None, +) NETWORK_CUT: Messages = ("E-NETWORK-CUT", "H-NETWORK-CUT", "A-NETWORK-CUT", None) NETWORK_UNREADABLE: Messages = ( "E-NETWORK-UNREADABLE", @@ -227,6 +525,22 @@ def _set_client(client: httpx.Client | None) -> None: "A-NETWORK-UNREADABLE", None, ) +NETWORK_REDIRECT: Messages = ( + "E-NETWORK-REDIRECT", + "H-NETWORK-REDIRECT", + "A-NETWORK-REDIRECT", + None, +) + +# auth_required's other message: the only key was in a user config file this +# AI agent session doesn't trust. next_step is null: a person +# has to fix the file. +KEY_UNTRUSTED: Messages = ( + "E-KEY-UNTRUSTED", + "W_CFG_UNTRUSTED_FILE_HINT", + "W_CFG_UNTRUSTED_FILE_HINT", + None, +) # Network failure reasons for E-NETWORK's {reason}. REASON_DNS = "name resolution failed" @@ -237,12 +551,13 @@ def _set_client(client: httpx.Client | None) -> None: REASON_TRUNCATED = "response cut off" REASON_UNREADABLE = "unreadable response" REASON_PROXY = "the proxy refused the connection" +REASON_REDIRECT = "unexpected redirect" REASON_OTHER = "connection failed" -KEY_SOURCE_ENV = "DEEPGRAM_API_KEY (env)" -KEY_SOURCE_FLAG = "--api-key" -KEY_SOURCE_KEYRING = "system keyring" -KEY_SOURCE_FILE = "config file" +KEY_SOURCE_ENV = credentials.KEY_SOURCE_ENV +KEY_SOURCE_FLAG = credentials.KEY_SOURCE_FLAG +KEY_SOURCE_KEYRING = credentials.KEY_SOURCE_KEYRING +KEY_SOURCE_FILE = credentials.KEY_SOURCE_FILE class TransportError(BaseException): @@ -272,6 +587,66 @@ def error(self) -> str: return STRINGS[self.messages[0]].format(**self.values) +class RedirectRefused(TransportError): + """A request was answered with a redirect, which is never followed. + + REST and WebSocket alike: the key and the request's audio or text go + only to the host that passed the host check. ``message`` and + ``exit_code`` read as on the ``click.ClickException`` it was in 0.4.1. + """ + + exit_code = 1 + + @property + def message(self) -> str: + return self.error + + +def redirect_refused( + url: str, + location: str | None, + *, + command: str = "dg", + payload: str | None = None, +) -> RedirectRefused: + """The error for a redirect from ``url`` to ``location``. + + Only origins are shown, never paths or queries, and anything a server + put in the Location header is made safe to print first. ``payload`` + names what a WebSocket would have sent ("audio", "text"); ``None`` for + REST, where the request reached ``url`` and nothing went further. + """ + from urllib.parse import urljoin + + from .hosts import parse_origin + from .messages import display + + here = parse_origin(url) + there = None + if location: + # Header bytes outside ASCII arrive as surrogates; show them as "?". + location = location.encode("ascii", "replace").decode("ascii") + try: + there = parse_origin(urljoin(url, location)) + except ValueError: # e.g. an unbalanced "[" in the host + there = None + host = _clean(here.host if here else url) + sent = "nothing was sent there" if payload is None else f"no {payload} was sent" + return RedirectRefused( + "network_unavailable", + { + "origin": display(here.origin if here else "?"), + "location_origin": display(there.origin if there else "?"), + "command": command, + "sent": sent, + "host": host, + "prog": prog() or "dg", + }, + {"host": host, "reason": REASON_REDIRECT}, + NETWORK_REDIRECT, + ) + + # ── Mapping ────────────────────────────────────────────────────────── _SECRET_URL = re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]*@") @@ -404,14 +779,27 @@ def map_exception( if isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError)): host = _host(exc) reason = _network_reason(exc) - cut = reason == REASON_TRUNCATED + messages: Messages | None = None + if reason == REASON_TRUNCATED: + messages = NETWORK_CUT + elif reason in (REASON_RESET, REASON_CLOSED) and isinstance( + exc, (httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError) + ): + # The connection was open and the request had started, so + # "couldn't reach" would be wrong. A reset while connecting + # (ConnectError) stays E-NETWORK. + messages = NETWORK_DROPPED return TransportError( "network_unavailable", {"host": host, "reason": reason}, {"host": host, "reason": reason}, - NETWORK_CUT if cut else None, + messages, ) + ws = _map_websocket(exc, secrets) + if ws is not None: + return ws + status = getattr(exc, "status_code", None) if not isinstance(status, int) or "deepgram" not in sys.modules: return None @@ -419,7 +807,6 @@ def map_exception( if not isinstance(exc, ApiError): return None - headers = exc.headers if 200 <= status < 300: # The SDK got a complete success response it couldn't parse: most # often not Deepgram's at all (a proxy's or captive portal's page). @@ -431,14 +818,37 @@ def map_exception( {"host": host, "reason": REASON_UNREADABLE}, NETWORK_UNREADABLE, ) + return for_status(status, exc.headers, exc.body, secrets) + + +def for_status( + status: int, + headers: object = None, + body: object = None, + secrets: tuple[str, ...] = (), + *, + host: str | None = None, + path: str | None = None, +) -> TransportError | None: + """The coded error for an HTTP error status, REST or WebSocket upgrade. + + ``host`` and ``path`` name the request when it didn't go through the + shared client (a WebSocket upgrade); else the last request's are used. + """ + if 300 <= status < 400: + # Credentialed requests don't follow redirects. + origin = host or _last_request.get("origin") or _host() + return redirect_refused( + origin, _header(headers, "location"), command=_running_command() + ) if status == 401: return TransportError("auth_invalid", {}, {"http_status": 401}) if status == 403: return TransportError("auth_forbidden", {}, {"http_status": 403}) if status == 404: - path = _clean(_last_request.get("path") or "this request") + where = _clean(path or _last_request.get("path") or "this request") return TransportError( - "not_found", {"path": path}, {"http_status": 404, "path": path} + "not_found", {"path": where}, {"http_status": 404, "path": where} ) if status == 429: details: dict[str, Any] = {"http_status": 429} @@ -451,7 +861,7 @@ def map_exception( "invalid_argument", { "http_status": status, - "api_message": _api_message(exc.body, status, secrets), + "api_message": _api_message(body, status, secrets), }, {"http_status": status}, ) @@ -464,10 +874,138 @@ def map_exception( return None +# Seconds a WebSocket handshake may take before it maps to ``timeout``. +WS_OPEN_TIMEOUT_S = 10.0 + + +def _ws_status(exc: BaseException) -> tuple[int, object, object] | None: + """(status, headers, body) of a refused WebSocket upgrade, if ``exc`` is one.""" + if "websockets" in sys.modules: + response = getattr(exc, "response", None) + status = getattr(response, "status_code", None) + if isinstance(status, int) and type(exc).__module__.startswith("websockets"): + headers = getattr(response, "headers", None) + body = getattr(response, "body", None) + if isinstance(body, bytes): + body = body.decode("utf-8", "replace") + return status, dict(headers or {}), body + status = getattr(exc, "status_code", None) # legacy InvalidStatusCode + if isinstance(status, int) and type(exc).__module__.startswith("websockets"): + return status, dict(getattr(exc, "headers", None) or {}), None + return None + + +def _timeout_types() -> tuple[type[BaseException], ...]: + types: list[type[BaseException]] = [TimeoutError] + if "asyncio" in sys.modules: + import asyncio + + types.append(asyncio.TimeoutError) # a separate class before 3.11 + return tuple(types) + + +def _map_websocket( + exc: BaseException, + secrets: tuple[str, ...], + *, + host: str | None = None, + path: str | None = None, + opening: bool = False, +) -> TransportError | None: + """A WebSocket open that failed: refused upgrade, network or timeout.""" + refused = _ws_status(exc) + if refused is not None: + status, headers, body = refused + return for_status(status, headers, body, secrets, host=host, path=path) + if not opening and "websockets" in sys.modules: + opening = type(exc).__module__.startswith("websockets") + if not opening: + return None + where = _clean(host) if host else _host(exc) + if any(isinstance(item, _timeout_types()) for item in _chain(exc)): + return TransportError( + "timeout", + {"host": where, "timeout_s": _seconds(WS_OPEN_TIMEOUT_S)}, + {"host": where, "timeout_s": _seconds(WS_OPEN_TIMEOUT_S)}, + ) + if any(isinstance(item, OSError) for item in _chain(exc)): + reason = _network_reason(exc) + return TransportError( + "network_unavailable", + {"host": where, "reason": reason}, + {"host": where, "reason": reason}, + ) + if "websockets" in sys.modules: + from websockets.exceptions import InvalidHandshake + + # The server answered the upgrade with something that isn't a valid + # HTTP response (a proxy, HTTP/1.0, a cut-off reply). + if any(isinstance(item, InvalidHandshake) for item in _chain(exc)): + return TransportError( + "network_unavailable", + {"host": where, "reason": REASON_UNREADABLE, "prog": prog() or "dg"}, + {"host": where, "reason": REASON_UNREADABLE}, + NETWORK_UNREADABLE, + ) + return None + + +def map_ws_open( + exc: BaseException, + secrets: tuple[str, ...] = (), + *, + host: str | None = None, + path: str | None = None, +) -> TransportError | None: + """The coded error for a WebSocket open to ``host`` that raised ``exc``. + + A refused upgrade maps by its HTTP status through the REST table; DNS, + connect and TLS failures map to ``network_unavailable``; an open that + timed out maps to ``timeout``. + """ + if isinstance(exc, TransportError): + return exc + return _map_websocket(exc, secrets, host=host, path=path, opening=True) + + def _seconds(value: float) -> int | float: return int(value) if float(value).is_integer() else value +def plain_detail(exc: BaseException) -> str: + """What went wrong, in the words of the error that caused it. + + The client wraps failures it can't map in the SDK's ``ApiError``, + whose text is a dump of its fields (``headers: None, status_code: + None, body: ...``). For one with no status, this is the message of + the error it was raised from; anything else is its own message. + """ + for item in _chain(exc): + if ( + type(item).__name__ == "ApiError" + and getattr(item, "status_code", 1) is None + ): + continue + text = str(item).strip() + if text: + return text + return type(item).__name__ + body = getattr(exc, "body", None) + return str(body) if body else type(exc).__name__ + + +def _running_command() -> str: + """The command being run, as typed (``dg listen``), for messages that + name it; ``dg`` outside a command.""" + if "click" in sys.modules: + import click + + ctx = click.get_current_context(silent=True) + if ctx is not None and ctx.command_path: + return ctx.command_path + return "dg" + + def raise_for( exc: BaseException, secrets: tuple[str, ...] = (), @@ -517,33 +1055,17 @@ def prog() -> str | None: def _key_source(auth_manager: AuthManager | None) -> str | None: + """Where the key that was sent came from: the resolver's answer, so the + error path reads the keyring no second time.""" if auth_manager is None: return None if auth_manager.explicit_api_key: return KEY_SOURCE_FLAG try: - key = auth_manager.get_api_key() + source = auth_manager.resolve().key_source except Exception: return None - if not key: - return None - # The order `dg whoami` uses: keyring, then the config file (a profile key - # equal to the env value came from the env), then the env. - import keyring - - from .auth import KEYRING_SERVICE - - env_key = os.environ.get("DEEPGRAM_API_KEY") - profile = auth_manager.config.profile or "default" - try: - stored = keyring.get_password(KEYRING_SERVICE, f"api-key.{profile}") - except Exception: - stored = None - if stored and stored == key: - return KEY_SOURCE_KEYRING - # Config merges the env key into the profile, so a file key equal to the - # env value came from the env. - return KEY_SOURCE_ENV if env_key == key else KEY_SOURCE_FILE + return None if source == credentials.KEY_SOURCE_NONE else source def render( @@ -564,7 +1086,12 @@ def render( next_step: str | None = None hint_id = human if not agentic else (automation or agent) if soft_only else agent - if err.code == "auth_invalid": + if err.code == "auth_required": + # Nobody can approve a device code under soft-only signals (CI, cron), + # and a person has to fix an untrusted file. + if not soft_only and program and err.messages[0] == "E-AUTH-REQUIRED": + next_step = f"{program} login" + elif err.code == "auth_invalid": source = _key_source(auth_manager) if source: details["key_source"] = source diff --git a/packages/deepctl-core/src/deepctl_core/websocket.py b/packages/deepctl-core/src/deepctl_core/websocket.py index cd4bf4b7..f3f13707 100644 --- a/packages/deepctl-core/src/deepctl_core/websocket.py +++ b/packages/deepctl-core/src/deepctl_core/websocket.py @@ -8,63 +8,25 @@ followed with the same headers (14 and 15, the key included), followed without the key (16 and later, the data still goes), or raised. These helpers turn every 3xx into :class:`RedirectRefused` before a second -connection is opened. REST calls through the SDK don't follow redirects -either (``follow_redirects=False`` in the client); :func:`redirect_refused` -words that error too, with no ``payload``. +connection is opened. REST calls through the shared client don't follow +redirects either; :func:`deepctl_core.transport.for_status` maps their 3xx +to the same :class:`RedirectRefused`. """ from __future__ import annotations from typing import Any -from urllib.parse import urljoin -import click +from .transport import RedirectRefused, redirect_refused -from .hosts import parse_origin -from .messages import display - -E_WS_REDIRECT = ( - "{origin} answered with a redirect to {location_origin}. {command} " - "doesn't follow redirects, so no {payload} was sent." -) -# REST: the request already reached the host that was checked; nothing -# goes to the new location. -E_HTTP_REDIRECT = ( - "{origin} answered with a redirect to {location_origin}. {command} " - "doesn't follow redirects, so nothing was sent there." -) - - -class RedirectRefused(click.ClickException): - """A request was answered with a redirect, which isn't followed.""" - - -def redirect_refused( - url: str, location: str | None, *, command: str, payload: str | None -) -> RedirectRefused: - """The error for a redirect from ``url`` to ``location``. - - Only origins are shown, never paths or queries, and anything a server - put in the Location header is made safe to print first. - """ - here = parse_origin(url) - there = None - if location: - # Header bytes outside ASCII arrive as surrogates; show them as "?". - location = location.encode("ascii", "replace").decode("ascii") - try: - there = parse_origin(urljoin(url, location)) - except ValueError: # e.g. an unbalanced "[" in the host - there = None - text = E_HTTP_REDIRECT if payload is None else E_WS_REDIRECT - return RedirectRefused( - text.format( - origin=display(here.origin if here else "?"), - location_origin=display(there.origin if there else "?"), - command=command, - payload=payload, - ) - ) +# The error and its wording live in transport (one redirect refusal for +# REST and WebSocket); re-exported here for the connect helpers' callers. +__all__ = [ + "RedirectRefused", + "connect_async", + "connect_sync", + "redirect_refused", +] def _as_refusal( diff --git a/packages/deepctl-core/tests/unit/test_auth.py b/packages/deepctl-core/tests/unit/test_auth.py index 946f1e11..14862b5a 100644 --- a/packages/deepctl-core/tests/unit/test_auth.py +++ b/packages/deepctl-core/tests/unit/test_auth.py @@ -9,6 +9,7 @@ import pytest from deepctl_core.auth import AuthenticationError, AuthManager from deepctl_core.config import Config +from deepctl_core.transport import request_headers @pytest.fixture @@ -70,7 +71,7 @@ def test_verify_credentials_success(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/test-project", headers={ - "Authorization": "Token sk-test-key", + **request_headers("sk-test-key"), "Content-Type": "application/json", }, ) @@ -160,7 +161,7 @@ def test_verify_credentials_no_project_id(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects", headers={ - "Authorization": "Token sk-test-key", + **request_headers("sk-test-key"), "Content-Type": "application/json", }, ) @@ -205,7 +206,7 @@ def test_verify_credentials_uses_stored_credentials( auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/stored-project", headers={ - "Authorization": "Token sk-stored-key", + **request_headers("sk-stored-key"), "Content-Type": "application/json", }, ) @@ -237,7 +238,7 @@ def test_verify_credentials_uses_env_vars(self, auth_manager): auth_manager.client.get.assert_called_once_with( "https://api.deepgram.com/v1/projects/env-project", headers={ - "Authorization": "Token sk-env-key", + **request_headers("sk-env-key"), "Content-Type": "application/json", }, ) @@ -602,7 +603,10 @@ class TestFiniteCredentialFlow: def _make_auth(self, mock_config): with patch("deepctl_core.auth.httpx.Client"): with patch("deepctl_core.auth.keyring"): - return AuthManager(mock_config) + auth = AuthManager(mock_config) + # The client is built on first use; build it under the patch. + _ = auth.client + return auth # ------------------------------------------------------------------ # _store_token — finite direct credential path diff --git a/packages/deepctl-core/tests/unit/test_auth_insecure.py b/packages/deepctl-core/tests/unit/test_auth_insecure.py index 88bfd1df..8a01d8b3 100644 --- a/packages/deepctl-core/tests/unit/test_auth_insecure.py +++ b/packages/deepctl-core/tests/unit/test_auth_insecure.py @@ -25,7 +25,8 @@ def _isolated(monkeypatch): def client_kwargs(config) -> dict: with patch.object(auth_module.httpx, "Client") as client: - AuthManager(config) + # The client is built on first use (login's device flow and verify). + _ = AuthManager(config).client return dict(client.call_args.kwargs) @@ -172,3 +173,87 @@ def test_other_loopback_addresses_keep_verification(capsys): _warn_if_verified(httpx.Request("GET", "https://127.0.0.2/")) assert "applies only to localhost" in capsys.readouterr().err + + +# -- The shared API client (SDK) and dg api's verified client -------------- + + +@pytest.fixture +def fresh_transport(monkeypatch): + from deepctl_core import transport + + monkeypatch.setattr(transport, "_client", None) + monkeypatch.setattr(transport, "_client_insecure", False) + monkeypatch.setattr(transport, "_verified", None) + return transport + + +def _verifies(client: httpx.Client, url: str) -> bool: + import ssl + + pool = client._transport_for_url(httpx.URL(url))._pool # type: ignore[attr-defined] + return pool._ssl_context.verify_mode == ssl.CERT_REQUIRED + + +@pytest.mark.parametrize("value", ["1", "true", "TRUE"]) +def test_shared_client_skips_verification_for_loopback_only(fresh_transport, monkeypatch, value): + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", value) + shared = fresh_transport.shared_client() + assert not _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert not _verifies(shared, "https://localhost:8443/v1/models") + assert not _verifies(shared, "https://[::1]:8443/v1/models") + for url in ( + "https://api.deepgram.com/v1/models", + "https://mitm.example:8443/v1/models", + "https://127.0.0.2:8443/v1/models", + ): + assert _verifies(shared, url), url + # dg api's client verifies every host, loopback included (as in 0.4.1). + verified = fresh_transport.verified_client() + assert verified is not shared + assert _verifies(verified, "https://127.0.0.1:8443/v1/projects") + assert verified.follow_redirects is False + + +@pytest.mark.parametrize("value", ["0", "false", "no", ""]) +def test_shared_client_verifies_for_other_values(fresh_transport, monkeypatch, value): + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", value) + shared = fresh_transport.shared_client() + assert _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert _verifies(shared, "https://mitm.example:8443/v1/models") + assert fresh_transport.verified_client() is shared # one client per process + + +def test_shared_client_verifies_for_agents(fresh_transport, monkeypatch, capsys): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + shared = fresh_transport.shared_client() + assert _verifies(shared, "https://127.0.0.1:8443/v1/models") + assert "Ignoring DEEPGRAM_CLI_INSECURE" in capsys.readouterr().err + + +def test_shared_client_loopback_goes_direct(fresh_transport, proxy, capsys): + server = _Recorder() + try: + try: + fresh_transport.shared_client().get( + f"https://127.0.0.1:{server.port}/v1/models", timeout=3 + ) + except httpx.HTTPError: + pass + finally: + server.close() + assert proxy.lines == [] + assert len(server.lines) == 1 # TLS ClientHello, straight to the server + + +def test_shared_client_remote_host_keeps_proxy_and_warns(fresh_transport, proxy, capsys): + for _ in range(2): + try: + fresh_transport.shared_client().get( + "https://api.deepgram.example/v1/models", timeout=3 + ) + except httpx.HTTPError: + pass + assert proxy.lines == ["CONNECT api.deepgram.example:443 HTTP/1.1"] * 2 + err = capsys.readouterr().err + assert err.count("DEEPGRAM_CLI_INSECURE applies only to localhost") == 1 diff --git a/packages/deepctl-core/tests/unit/test_client.py b/packages/deepctl-core/tests/unit/test_client.py index c6bfeb68..ac8b104a 100644 --- a/packages/deepctl-core/tests/unit/test_client.py +++ b/packages/deepctl-core/tests/unit/test_client.py @@ -72,10 +72,13 @@ def test_client_property_creates_client( # Verify auth guard was called mock_auth_manager.guard.assert_called_once() - # Verify DGClient was created with keyword args, without redirects + # Verify DGClient was created with keyword args, on the shared + # client, which follows no redirects assert mock_dg_client.called assert mock_dg_client.call_args.kwargs["api_key"] == "sk-test" - assert mock_dg_client.call_args.kwargs["follow_redirects"] is False + http = mock_dg_client.call_args.kwargs["httpx_client"] + assert http is transport.shared_client() + assert http.follow_redirects is False # Verify client is cached assert client._client == mock_instance @@ -130,8 +133,8 @@ def test_create_client_with_custom_base_url( mock_dg_client.assert_called_once_with( api_key="sk-test", httpx_client=transport.shared_client(), + session_id=transport.session_id(), environment=mock_env.return_value, - follow_redirects=False, ) @patch("deepctl_core.client.DGClient") diff --git a/packages/deepctl-core/tests/unit/test_config_location.py b/packages/deepctl-core/tests/unit/test_config_location.py index a7dfbb9a..9b010c1a 100644 --- a/packages/deepctl-core/tests/unit/test_config_location.py +++ b/packages/deepctl-core/tests/unit/test_config_location.py @@ -787,10 +787,21 @@ def test_missing_key_after_untrusted_file_key_says_so(real_home, monkeypatch, ca "above), and no other API key is set." ) assert str(err.value) == expected - captured = capsys.readouterr() - text = (captured.out + captured.err).replace("\n", " ") - assert f"ERROR: {expected}" in text - assert f"HINT: {FILE_HINT}" in text + # guard() raises the coded auth_required (E-KEY-UNTRUSTED); + # BaseCommand prints its two lines (test_credentials.py covers that). + from deepctl_core import transport + + assert isinstance(err.value, transport.TransportError) + assert err.value.code == "auth_required" + output._output_config.update(agentic=True) + try: + result = transport.render(err.value, command_path="listen") + finally: + output._output_config.update(agentic=False) + assert result.error == expected + assert result.hint == FILE_HINT + assert result.next_step is None + text = f"{result.error} {result.hint}" assert "not set in the configuration file" not in text assert "deepctl login" not in text assert "dg login" not in text @@ -921,6 +932,16 @@ def deny(*args, **kwargs): assert len(texts) == 1 and "set DEEPGRAM_API_KEY instead" in texts[0] +def _render_guard_error(manager, error) -> None: + """The two stderr lines BaseCommand prints for guard()'s coded error.""" + from deepctl_core import transport + from rich.markup import escape + + result = transport.render(error, command_path="projects", auth_manager=manager) + output.print_error(escape(result.error)) + output.print_hint(escape(result.hint)) + + def _run_projects_guard(capsys): """Config load + notices + guard, as a `dg projects` run renders them.""" from deepctl_core.auth import AuthenticationError, AuthManager @@ -929,8 +950,10 @@ def _run_projects_guard(capsys): try: config = Config() config_module.flush_config_notices() - with pytest.raises(AuthenticationError): - AuthManager(config).guard() + manager = AuthManager(config) + with pytest.raises(AuthenticationError) as raised: + manager.guard() + _render_guard_error(manager, raised.value) finally: output._output_config.update(agentic=False) return capsys.readouterr().err @@ -973,8 +996,10 @@ def test_untrusted_hint_follows_the_error_when_nothing_warned(real_home, monkeyp plant_with_key(real_home, 0o666) output._output_config.update(agentic=True) try: - with pytest.raises(AuthenticationError): - AuthManager(Config()).guard() + manager = AuthManager(Config()) + with pytest.raises(AuthenticationError) as raised: + manager.guard() + _render_guard_error(manager, raised.value) finally: output._output_config.update(agentic=False) lines = [line for line in capsys.readouterr().err.splitlines() if line.strip()] diff --git a/packages/deepctl-core/tests/unit/test_credentials.py b/packages/deepctl-core/tests/unit/test_credentials.py new file mode 100644 index 00000000..3cf3a01e --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_credentials.py @@ -0,0 +1,630 @@ +"""deepctl_core.credentials: one resolver, 0.4.2's precedence.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +import yaml +from deepctl_core import auth as auth_module +from deepctl_core import config as config_module +from deepctl_core import credentials, hosts, output, transport +from deepctl_core.auth import AuthManager, AuthRequiredError +from deepctl_core.config import Config + +pytestmark = pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="XDG layout (Linux)" +) + +KEYRING_SERVICE = auth_module.KEYRING_SERVICE + + +class FakeKeyring: + """A dict-backed keyring; ``hang`` makes reads block until released.""" + + def __init__(self, entries: dict[str, str] | None = None) -> None: + self.entries = dict(entries or {}) + self.reads: list[str] = [] + self.deletes: list[str] = [] + self.hang: threading.Event | None = None + + def get_password(self, service: str, username: str) -> str | None: + assert service == KEYRING_SERVICE + self.reads.append(username) + if self.hang is not None: + self.hang.wait(10) + return self.entries.get(username) + + def delete_password(self, service: str, username: str) -> None: + self.deletes.append(username) + self.entries.pop(username, None) + + def set_password(self, service: str, username: str, value: str) -> None: + self.entries[username] = value + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setattr(config_module, "_pending_notices", []) + monkeypatch.setattr(config_module, "_seen_notices", set()) + monkeypatch.setattr(hosts, "_reported", set()) + home = tmp_path / "home" + home.mkdir() + monkeypatch.setattr(config_module, "_real_home", lambda: str(home)) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + for name in output.AI_AGENT_ENV_MARKERS: + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(output, "is_ai_agent", lambda: False) + monkeypatch.setattr(transport, "prog", lambda: "dg") + return home + + +@pytest.fixture +def kr(monkeypatch) -> FakeKeyring: + fake = FakeKeyring() + monkeypatch.setattr(auth_module, "keyring", fake, raising=False) + return fake + + +def write_config(home: Path, data: dict[str, Any], mode: int = 0o600) -> Path: + directory = home / ".config" / "deepctl" + directory.mkdir(parents=True, exist_ok=True) + path = directory / "config.yaml" + path.write_text(yaml.safe_dump(data)) + os.chmod(path, mode) + return path + + +# ── 0.4.2's get_api_key(), verbatim in substance, as the golden ───────── + + +def golden_042(config: Config, explicit: str | None, kr: FakeKeyring) -> str | None: + if explicit: + return explicit + profile_name = config.profile or "default" + current_profile = config.get_profile(profile_name) + expired, _ = credentials.credential_expiry(config, profile_name) + if not expired: + api_key = kr.entries.get(f"api-key.{profile_name}") + if api_key: + return api_key + if current_profile.api_key: + return current_profile.api_key + return os.getenv("DEEPGRAM_API_KEY") or None + + +def whoami_source_042(config: Config, key: str | None, kr: FakeKeyring) -> str: + """`dg whoami`'s key_source rule (0.4.2), plus the flag.""" + if key is None: + return credentials.KEY_SOURCE_NONE + name = config.profile or "default" + if kr.entries.get(f"api-key.{name}") == key: + return credentials.KEY_SOURCE_KEYRING + env = os.getenv("DEEPGRAM_API_KEY") + if config.get_profile(name).api_key == key: + return credentials.KEY_SOURCE_ENV if env == key else credentials.KEY_SOURCE_FILE + return credentials.KEY_SOURCE_ENV + + +PROFILE_SOURCES = ("none", "--profile", "DEEPGRAM_PROFILE", "user-file default") + + +@pytest.mark.parametrize("profile_source", PROFILE_SOURCES) +@pytest.mark.parametrize("flag", [False, True]) +@pytest.mark.parametrize("keyring_set", [False, True]) +@pytest.mark.parametrize("file_set", [False, True]) +@pytest.mark.parametrize("env_set", [False, True]) +def test_precedence_matches_042( + home, kr, monkeypatch, profile_source, flag, keyring_set, file_set, env_set +): + profile = "default" if profile_source == "none" else "work" + data: dict[str, Any] = {"profiles": {}} + if file_set: + data["profiles"][profile] = {"api_key": "f" * 39 + "1"} + if profile_source == "user-file default": + data["default_profile"] = "work" + write_config(home, data) + if keyring_set: + kr.entries[f"api-key.{profile}"] = "f" * 39 + "2" + if env_set: + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 39 + "3") + if profile_source == "DEEPGRAM_PROFILE": + monkeypatch.setenv("DEEPGRAM_PROFILE", "work") + config = Config(profile="work" if profile_source == "--profile" else None) + explicit = "f" * 39 + "4" if flag else None + + resolved = AuthManager(config, explicit).resolve() + + want = golden_042(config, explicit, kr) + assert resolved.api_key == want + assert resolved.profile == profile + assert resolved.profile_source == profile_source + if flag: + assert resolved.key_source == credentials.KEY_SOURCE_FLAG + assert kr.reads == [] # --api-key never reads the keyring + else: + assert resolved.key_source == whoami_source_042(config, want, kr) + + +def test_keyring_read_once_per_run(home, kr, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + auth = AuthManager(Config()) + auth.guard() + auth.get_api_key() + auth.get_credential_source() + auth.has_profile_credentials() + auth.is_authenticated() + auth.get_project_id() + assert transport._key_source(auth) == credentials.KEY_SOURCE_ENV + assert kr.reads == ["api-key.default"] + + +def test_login_forgets_the_cached_answer(home, kr, monkeypatch): + auth = AuthManager(Config()) + assert auth.get_api_key() is None + monkeypatch.setattr(auth, "verify_credentials", lambda *a: (True, "ok", None)) + auth.login_with_api_key("f" * 40) + assert auth.get_api_key() == "f" * 40 + + +def test_project_id_precedence(home, kr, monkeypatch): + write_config(home, {"profiles": {"default": {"project_id": "p-file"}}}) + monkeypatch.setenv("DEEPGRAM_PROJECT_ID", "p-env") + # Config merges the env value over the file's, as in 0.4.2. + assert AuthManager(Config()).get_project_id() == "p-env" + assert AuthManager(Config(), None, "p-flag").get_project_id() == "p-flag" + monkeypatch.delenv("DEEPGRAM_PROJECT_ID") + assert AuthManager(Config()).get_project_id() == "p-file" + assert kr.reads == [] # project questions don't read the keyring + + +# ── Where the credentials line says they came from ───────────────────── + + +def test_env_key_is_reported_as_the_environments(home, kr, monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + auth = AuthManager(Config()) + # 0.4.2 said "profile 'default'": Config merges the env key into the profile. + assert auth.get_credential_source() == "environment variables" + assert auth.has_profile_credentials() == (True, False) # 0.4.2's answer + + +def test_stored_key_is_reported_as_the_profiles(home, kr, monkeypatch): + kr.entries["api-key.default"] = "f" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert AuthManager(Config()).get_credential_source() == "profile 'default'" + write_config(home, {"profiles": {"default": {"api_key": "d" * 40}}}) + kr.entries.clear() + monkeypatch.delenv("DEEPGRAM_API_KEY") + assert AuthManager(Config()).get_credential_source() == "profile 'default'" + assert AuthManager(Config(), "c" * 40).get_credential_source() == "explicit flags" + + +# ── Trust cells and key expiry ──────────────────────────────── + + +@pytest.mark.parametrize("mode", [0o644, 0o666, 0o660]) +@pytest.mark.parametrize("agent", [False, True]) +def test_trust_cells(home, kr, monkeypatch, mode, agent): + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, mode) + resolved = AuthManager(Config()).resolve() + writable = bool(mode & 0o022) + if agent and writable: + assert resolved.api_key is None + assert resolved.untrusted_source == "~/.config/deepctl/config.yaml" + with pytest.raises(AuthRequiredError) as err: + AuthManager(Config()).guard() + assert err.value.messages == transport.KEY_UNTRUSTED + else: + assert resolved.api_key == "f" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_FILE + assert resolved.untrusted_source is None + + +def test_untrusted_file_with_env_key_uses_env(home, kr, monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, 0o666) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + resolved = AuthManager(Config()).resolve() + assert resolved.api_key == "e" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_ENV + + +def test_sf1_untrusted_expiry_never_deletes_keyring(home, kr, monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config( + home, + {"profiles": {"default": {"api_key": "f" * 40, "dg_token_expires_at": past}}}, + 0o666, + ) + kr.entries["api-key.default"] = "k" * 40 + AuthManager(Config()).resolve() + assert kr.deletes == [] + assert kr.entries["api-key.default"] == "k" * 40 + + +def test_trusted_expiry_still_deletes_keyring(home, kr): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config(home, {"profiles": {"default": {"dg_token_expires_at": past}}}) + kr.entries["api-key.default"] = "k" * 40 + with pytest.raises(auth_module.AuthenticationError, match="expired"): + AuthManager(Config()).get_api_key() + assert kr.deletes == ["api-key.default"] + + +def test_introspection_never_prunes(home, kr, monkeypatch): + """Whoami and login's checks stay read-only, as in 0.4.2.""" + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + write_config(home, {"profiles": {"default": {"dg_token_expires_at": past}}}) + kr.entries["api-key.default"] = "k" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + auth = AuthManager(Config()) + assert auth.has_profile_credentials() == (False, False) + assert auth.has_profile_credentials("default") == (False, False) + assert auth.is_authenticated() is True + assert kr.deletes == [] + # A command that uses the key prunes it, once (0.4.2's get_api_key). + assert auth.get_api_key() == "e" * 40 + assert auth.get_api_key() == "e" * 40 + assert kr.deletes == ["api-key.default"] + + +def test_one_resolution_per_profile_spelling(home, kr): + kr.entries["api-key.default"] = "k" * 40 + auth = AuthManager(Config()) + auth.resolve(profile="default", use_flags=False, prune=False) + auth.resolve() + auth.has_profile_credentials() + assert kr.reads == ["api-key.default"] + + +# ── Keyring bound ─────────────────────────────────────────────────────── + + +def test_keyring_bound_falls_back_once(home, kr, monkeypatch, capsys): + # A 2 s bound: the first read waits it once, a second wait would show. + monkeypatch.setattr(credentials, "KEYRING_BOUND_S", 2.0) + monkeypatch.setattr(credentials, "_interactive", lambda: False) + monkeypatch.setattr(output.stderr_console, "_width", 400) + kr.entries["api-key.default"] = "k" * 40 + kr.hang = threading.Event() + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + try: + started = time.monotonic() + assert AuthManager(Config()).get_api_key() == "e" * 40 + assert time.monotonic() - started < 6.0 + # Cached for the rest of the process: no second wait, no second line. + started = time.monotonic() + assert AuthManager(Config()).get_api_key() == "e" * 40 + assert time.monotonic() - started < 1.0 + finally: + kr.hang.set() + err = capsys.readouterr().err + assert err.count("The system keyring didn't answer within 2 s") == 1 + assert "k" * 40 not in err and "e" * 40 not in err + assert len(kr.reads) == 1 + + +def test_keyring_unbounded_at_a_terminal(home, kr, monkeypatch): + monkeypatch.setattr(credentials, "_interactive", lambda: True) + + def no_threads(*args: Any, **kwargs: Any) -> None: + raise AssertionError("an interactive read runs in the caller's thread") + + monkeypatch.setattr(credentials.threading, "Thread", no_threads) + kr.entries["api-key.default"] = "k" * 40 + assert AuthManager(Config()).get_api_key() == "k" * 40 + + +def test_keyring_errors_fall_through(home, kr, monkeypatch): + def broken(*args: Any) -> None: + raise RuntimeError("no backend") + + monkeypatch.setattr(kr, "get_password", broken) + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert AuthManager(Config()).get_api_key() == "e" * 40 + + +# ── Placeholders ──────────────────────────────────────────────────────── + + +def test_placeholder_key_counts_as_unset(home, kr, monkeypatch, capsys): + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setenv("DEEPGRAM_API_KEY", "${DEEPGRAM_API_KEY}") + assert credentials.drop_env_placeholders() == ["DEEPGRAM_API_KEY"] + auth = AuthManager(Config()) + with pytest.raises(AuthRequiredError): + auth.guard() + credentials.flush_warnings() + err = capsys.readouterr().err + assert err.count("ignored DEEPGRAM_API_KEY: it holds a placeholder") == 1 + assert "${DEEPGRAM_API_KEY}" not in err + + +def test_placeholder_merged_before_the_drop_keeps_the_file_key(home, kr, monkeypatch): + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}) + monkeypatch.setenv("DEEPGRAM_API_KEY", "${DEEPGRAM_API_KEY}") + config = Config() # e.g. `dg mcp`'s handoff builds its own Config first + resolved = AuthManager(config).resolve() + assert resolved.api_key == "f" * 40 + assert resolved.key_source == credentials.KEY_SOURCE_FILE + + +@pytest.mark.parametrize( + "name", ["DEEPGRAM_PROFILE", "DEEPGRAM_PROJECT_ID", "DEEPGRAM_BASE_URL"] +) +def test_other_placeholders_count_as_unset(home, kr, monkeypatch, capsys, name): + monkeypatch.setattr(output.stderr_console, "_width", 400) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}) + monkeypatch.setenv(name, "${" + name + "}") + credentials.drop_env_placeholders() + config = Config() + resolved = AuthManager(config).resolve() + assert resolved.api_key == "f" * 40 + assert resolved.profile == "default" + assert resolved.project_id is None + assert hosts.check_credential_host(config) == hosts.DEFAULT_BASE_URL + credentials.flush_warnings() + credentials.flush_warnings() + err = capsys.readouterr().err + assert err.count(f"ignored {name}: it holds a placeholder") == 1 + + +def test_profile_placeholder_merged_before_the_drop(home, kr, monkeypatch): + write_config( + home, + {"default_profile": "work", "profiles": {"work": {"api_key": "f" * 40}}}, + ) + monkeypatch.setenv("DEEPGRAM_PROFILE", "${DEEPGRAM_PROFILE}") + resolved = AuthManager(Config()).resolve() + assert resolved.profile == "work" + assert resolved.api_key == "f" * 40 + + +# ── Env-vs-stored mismatch: implemented, off in 0.5.0 ────────────────── + + +def test_mismatch_check_is_off(home, kr, monkeypatch): + kr.entries["api-key.default"] = "k" * 40 + monkeypatch.setenv("DEEPGRAM_API_KEY", "e" * 40) + assert credentials.WARN_ENV_MISMATCH is False + assert AuthManager(Config()).resolve().env_mismatch is False + monkeypatch.setattr(credentials, "WARN_ENV_MISMATCH", True) + assert AuthManager(Config()).resolve().env_mismatch is True + assert credentials.fingerprint("k" * 40) != "k" * 40 + + +# ── auth_required rendering ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("agentic", "hard", "hint_id", "next_step"), + [ + (False, False, "H-AUTH-REQUIRED", "dg login"), + (True, True, "A-AUTH-REQUIRED", "dg login"), + (True, False, "C-AUTH-REQUIRED", None), + ], +) +def test_auth_required_render(monkeypatch, agentic, hard, hint_id, next_step): + monkeypatch.setattr(output, "is_ai_agent", lambda: hard) + monkeypatch.setitem(output._output_config, "agentic", agentic) + result = transport.render(AuthRequiredError(), command_path="listen") + assert result.code == "auth_required" + assert result.error == "DEEPGRAM_API_KEY is not set" + assert result.hint == transport.STRINGS[hint_id].format(prog="dg") + assert result.next_step == next_step + assert result.model_dump()["schema_version"] == 1 + assert "details" not in result.model_dump() + + +def test_key_untrusted_render_has_no_next_step(monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + err = AuthRequiredError(untrusted_source="~/.config/deepctl/config.yaml") + result = transport.render(err, command_path="listen") + assert result.error == ( + "dg didn't use the API key in ~/.config/deepctl/config.yaml (see the " + "warning above), and no other API key is set." + ) + assert result.hint == ( + "A person must fix this file. If you are an AI agent, ask the user." + ) + assert result.next_step is None + + +def test_catalogue_text_is_0_4_1s(monkeypatch): + from deepctl_core import messages + + assert transport.STRINGS["E-KEY-UNTRUSTED"] == messages.E_KEY_UNTRUSTED + assert ( + transport.STRINGS["W_CFG_UNTRUSTED_FILE_HINT"] + == messages.W_CFG_UNTRUSTED_FILE_HINT + ) + assert isinstance(AuthRequiredError(), auth_module.AuthenticationError) + + +def _run_command(monkeypatch, capsys, *, requires_project: bool = False) -> Any: + import click + from deepctl_core.base_command import BaseCommand + from deepctl_core.models import BaseResult + + class NeedsAuth(BaseCommand): + name = "listen" + help = "test command" + requires_auth = True + + def handle(self, config, auth_manager, client, **kwargs): # type: ignore[no-untyped-def] + return BaseResult(status="success", message="done") + + NeedsAuth.requires_project = requires_project + monkeypatch.setattr(output.stderr_console, "_width", 400) + monkeypatch.setattr("deepctl_core.output.get_output_format", lambda: "json") + ctx = click.Context( + click.Command("listen"), info_name="dg listen", obj={"config": Config()} + ) + code = 0 + try: + NeedsAuth().execute(ctx) + except SystemExit as exc: + code = int(exc.code or 0) + except click.ClickException: + code = 1 + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def test_no_key_prints_coded_result_and_two_lines(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 1 + payload = json.loads(out) + assert payload == { + "error": "DEEPGRAM_API_KEY is not set", + "status": "error", + "code": "auth_required", + "hint": transport.STRINGS["A-AUTH-REQUIRED"].format(prog="dg"), + "next_step": "dg login", + "schema_version": 1, + } + assert err.splitlines() == [ + "ERROR: DEEPGRAM_API_KEY is not set", + "HINT: " + transport.STRINGS["A-AUTH-REQUIRED"].format(prog="dg"), + ] + + +def test_untrusted_file_prints_two_lines(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + write_config(home, {"profiles": {"default": {"api_key": "f" * 40}}}, 0o666) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 1 + payload = json.loads(out) + assert payload["code"] == "auth_required" + assert payload["next_step"] is None + lines = err.splitlines() + # The warning's hint prints once per run (0.4.1), before the error. + assert lines[-1].startswith("ERROR: dg didn't use the API key in ") + assert lines[-2] == ( + "HINT: A person must fix this file. If you are an AI agent, ask the user." + ) + assert err.count("HINT: ") == 1 + assert "f" * 40 not in out + err + + +def test_env_key_info_line(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + code, out, err = _run_command(monkeypatch, capsys) + assert code == 0 + assert err.splitlines() == ["INFO: Using credentials from environment variables"] + + +def test_no_project_warning_only_when_needed(home, kr, monkeypatch, capsys): + from deepctl_core import base_command + + monkeypatch.setitem(output._output_config, "agentic", True) + monkeypatch.setattr(base_command, "_agentic", True) + monkeypatch.setenv("DEEPGRAM_API_KEY", "f" * 40) + _, _, err = _run_command(monkeypatch, capsys, requires_project=True) + assert "WARN: No project ID specified" in err + + +# ── Import cost: no keyring with --api-key, no client in a group's --help ─ + + +def _imports(code: str, home: Path) -> set[str]: + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("DEEPGRAM_") and k not in output.AI_AGENT_ENV_MARKERS + } + env.update(HOME=str(home), DEEPCTL_TELEMETRY_DISABLED="1") + done = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + text=True, + env=env, + timeout=60, + ) + assert done.returncode == 0, done.stderr + return set(json.loads(done.stdout.strip().splitlines()[-1])) + + +def test_api_key_flag_never_imports_keyring(home): + modules = _imports( + """ + import json, sys + from deepctl_core.auth import AuthManager + from deepctl_core.config import Config + auth = AuthManager(Config(), "f" * 40) + auth.guard() + assert auth.get_api_key() == "f" * 40 + auth.get_credential_source() + print(json.dumps(sorted(sys.modules))) + """, + home, + ) + assert "keyring" not in modules + assert "httpx" not in modules + + +def test_group_subcommand_help_builds_no_client(home): + modules = _imports( + """ + import json, sys + import click + from deepctl_core.base_command import BaseCommand + from deepctl_core.base_group_command import BaseGroupCommand + + class Sub(BaseCommand): + name = "sub" + help = "sub" + def handle(self, config, auth_manager, client, **kwargs): + return None + + class Group(BaseGroupCommand): + name = "grp" + help = "grp" + def setup_commands(self): + return [click.Command("sub", callback=lambda: None, help="sub")] + + @click.group() + @click.pass_context + def root(ctx): + from deepctl_core.config import Config + ctx.obj = {"config": Config()} + + root.add_command(Group().get_click_group()) + # Not click.testing: on 3.14 it imports pdb, and with it asyncio and ssl. + try: + root.main(["grp", "sub", "--help"], prog_name="dg") + except SystemExit as exc: + assert exc.code == 0, exc.code + print(json.dumps(sorted(sys.modules))) + """, + home, + ) + for name in ("httpx", "keyring", "ssl", "deepgram"): + assert name not in modules, name diff --git a/packages/deepctl-core/tests/unit/test_output_channels.py b/packages/deepctl-core/tests/unit/test_output_channels.py index 8d9bb851..a1545daf 100644 --- a/packages/deepctl-core/tests/unit/test_output_channels.py +++ b/packages/deepctl-core/tests/unit/test_output_channels.py @@ -218,7 +218,8 @@ def test_explicit_flags_leave_stdout_parseable(self, capsys) -> None: payload = json.loads(captured.out) assert payload["status"] == "success" assert "Using credentials from" in captured.err - assert "No project ID specified" in captured.err + # Only commands that need a project say it's missing. + assert "No project ID specified" not in captured.err assert "Using credentials from" not in captured.out def test_project_id_line_also_goes_to_stderr(self, capsys) -> None: diff --git a/packages/deepctl-core/tests/unit/test_transport.py b/packages/deepctl-core/tests/unit/test_transport.py index 3b980e6c..03fb23f5 100644 --- a/packages/deepctl-core/tests/unit/test_transport.py +++ b/packages/deepctl-core/tests/unit/test_transport.py @@ -74,7 +74,7 @@ def record(request: httpx.Request) -> httpx.Response: seen.append(request) return handler(request) - # Same redirect setting as transport.shared_client(). + # _set_client applies shared_client()'s settings (no redirects, no cookies). transport._set_client( httpx.Client(transport=httpx.MockTransport(record), follow_redirects=False) ) @@ -228,11 +228,112 @@ def test_network_rows(factory, reason): assert err.error == ( "The connection to api.deepgram.com dropped before the response finished." ) + elif reason in ("connection reset", "connection closed"): + # Post-connect: the request had started. + assert err.messages == transport.NETWORK_DROPPED + assert err.error == ( + "The connection to api.deepgram.com dropped before the response " + f"was complete ({reason})." + ) else: assert err.messages == transport.CODES["network_unavailable"] assert err.error == f"Couldn't reach api.deepgram.com: {reason}." +@pytest.mark.parametrize( + ("factory", "reason", "messages"), + [ + # A reset or close while connecting is still "couldn't reach". + ( + _chained( + httpx.ConnectError, ConnectionResetError(54, "Connection reset by peer") + ), + "connection reset", + "E-NETWORK", + ), + ( + lambda r: httpx.ConnectError("Connection reset by peer", request=r), + "connection reset", + "E-NETWORK", + ), + # TLS failing on a read is a TLS failure, not a drop. + ( + _chained(httpx.ReadError, ssl.SSLError(1, "[SSL] record layer failure")), + "TLS handshake failed", + "E-NETWORK", + ), + # After connect: the upload was being written, or the response read. + ( + _chained(httpx.WriteError, BrokenPipeError(32, "Broken pipe")), + "connection reset", + "E-NETWORK-DROPPED", + ), + ( + _chained( + httpx.WriteError, ConnectionResetError(104, "Connection reset by peer") + ), + "connection reset", + "E-NETWORK-DROPPED", + ), + ( + lambda r: httpx.ReadError("", request=r), + "connection reset", + "E-NETWORK-DROPPED", + ), + ], +) +def test_dropped_only_after_connect(factory, reason, messages): + err = mapped(raising(factory)) + assert err.code == "network_unavailable" + assert err.details == {"host": "api.deepgram.com", "reason": reason} + assert err.messages[0] == messages + + +def _reset_after_full_body() -> tuple[int, threading.Thread]: + """A loopback server that reads the whole request, then resets.""" + import struct + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + + def serve() -> None: + conn, _ = listener.accept() + data = b"" + while b"\r\n\r\n" not in data: + data += conn.recv(65536) + head, _, body = data.partition(b"\r\n\r\n") + length = next( + int(line.split(b":", 1)[1]) + for line in head.split(b"\r\n") + if line.lower().startswith(b"content-length:") + ) + while len(body) < length: + body += conn.recv(65536) + conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + conn.close() + listener.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return listener.getsockname()[1], thread + + +def test_reset_after_the_full_body_is_dropped_not_unreachable(): + """The whole upload arrived, then the server reset. That isn't + "couldn't reach": the request had started (E-NETWORK-DROPPED).""" + port, thread = _reset_after_full_body() + with httpx.Client() as client, pytest.raises(httpx.TransportError) as raised: + client.post(f"http://127.0.0.1:{port}/v1/listen", content=b"x" * 3145729) + thread.join(5) + err = transport.map_exception(raised.value) + assert err is not None and err.code == "network_unavailable" + # macOS raises ReadError (reset); Linux may see EOF (closed) instead. + assert err.details["reason"] in ("connection reset", "connection closed") + assert err.messages == transport.NETWORK_DROPPED + assert err.error.startswith("The connection to 127.0.0.1 dropped before") + + def test_unreadable_2xx_hint_names_host_and_prog(monkeypatch): err = mapped(status(200, b"sign in")) payload = _render(err, monkeypatch, agentic=False, hard=False) @@ -308,14 +409,70 @@ def broken(*args, **kwargs): def test_every_code_is_catalogued(): for code, ids in [ *transport.CODES.items(), + ("dropped", transport.NETWORK_DROPPED), ("cut", transport.NETWORK_CUT), ("unreadable", transport.NETWORK_UNREADABLE), + ("redirect", transport.NETWORK_REDIRECT), ]: for message_id in ids: if message_id: assert message_id in transport.STRINGS, (code, message_id) +# What dg prints for a refused redirect: 0.4.1's text, origins only. +_REDIRECT_CASES = [ + ( + {"command": "dg listen"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg listen doesn't follow redirects, so nothing " + "was sent there.", + ), + ( + {"command": "dg listen", "payload": "audio"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg listen doesn't follow redirects, so no audio " + "was sent.", + ), + ( + {"command": "dg speak", "payload": "text"}, + "https://api.deepgram.com answered with a redirect to " + "https://evil.example. dg speak doesn't follow redirects, so no text " + "was sent.", + ), +] + + +def _refused(**kwargs: Any) -> RedirectRefused: + return transport.redirect_refused( + "https://api.deepgram.com/v1/listen?model=nova-3", + "https://evil.example/steal?k=1", + **kwargs, + ) + + +@pytest.mark.parametrize(("kwargs", "printed"), _REDIRECT_CASES) +def test_redirect_refusal_prints_e_network_redirect(kwargs, printed): + """The printed refusal is E-NETWORK-REDIRECT filled in, so the catalogue + entry that holds that string is the text users see.""" + err = _refused(**kwargs) + assert err.message == err.error == printed + assert err.messages[0] == "E-NETWORK-REDIRECT" + assert transport.STRINGS["E-NETWORK-REDIRECT"].format(**err.values) == printed + + +@pytest.mark.skipif( + not os.environ.get("DG_ERROR_CODES_JSON"), + reason="DG_ERROR_CODES_JSON names no error catalogue file", +) +@pytest.mark.parametrize(("kwargs", "printed"), _REDIRECT_CASES) +def test_catalogue_redirect_text_is_what_dg_prints(kwargs, printed): + catalogue = json.loads(Path(os.environ["DG_ERROR_CODES_JSON"]).read_text()) + template = catalogue["strings"]["E-NETWORK-REDIRECT"] + for name in ("origin", "location_origin", "command", "sent"): + assert "{%s}" % name in catalogue["placeholders"], name + assert template.format(**_refused(**kwargs).values) == printed + + @pytest.mark.skipif( not os.environ.get("DG_ERROR_CODES_JSON"), reason="DG_ERROR_CODES_JSON names no error catalogue file", @@ -343,13 +500,26 @@ def test_strings_match_catalogue(): assert message.get("hint_automation") == automation entry = catalogue["codes"]["network_unavailable"] for extra, reason in ( + (transport.NETWORK_DROPPED, transport.REASON_RESET), + (transport.NETWORK_DROPPED, transport.REASON_CLOSED), (transport.NETWORK_CUT, "response cut off"), (transport.NETWORK_UNREADABLE, "unreadable response"), + (transport.NETWORK_REDIRECT, transport.REASON_REDIRECT), ): message = next(m for m in entry["messages"] if m["id"] == extra[0]) assert (message["hint_human"], message["hint_agent"]) == extra[1:3] assert "hint_automation" not in message - assert entry["details_values"][extra[0]] == {"reason": [reason]} + assert reason in entry["details_values"][extra[0]]["reason"] + # A 3xx with credentials isn't followed, and has its own message. + assert transport.REASON_REDIRECT not in ( + entry["details_values"]["E-NETWORK-UNREADABLE"]["reason"] + ) + assert catalogue["transport_mapping"][0][0].split("|")[-1] == "3xx" + for message_id in ("W-ENV-PLACEHOLDER", "W-KEYRING-TIMEOUT"): + from deepctl_core import credentials + + text = getattr(credentials, message_id.replace("-", "_")) + assert catalogue["strings"][message_id] == text env = catalogue["codes"]["auth_invalid"]["env_key_variant"] assert env == { "hint_agent": "A-AUTH-INVALID-ENV", @@ -734,10 +904,11 @@ def handler(request): return httpx.Response(200, json={"models": []}) client, _ = facade(handler) - with pytest.raises(RedirectRefused): + with pytest.raises(TransportError) as err: client.list_models() assert [r.url.host for r in seen] == ["api.deepgram.com"] assert seen[0].headers["Authorization"] == f"Token {KEY}" + assert err.value.details["reason"] == "unexpected redirect" def test_shared_client_is_lazy_and_single(): @@ -758,3 +929,321 @@ def test_print_warning_ignore_quiet(ignore_quiet, monkeypatch, capsys): monkeypatch.setitem(output._output_config, "agentic", True) output.print_warning("needed", stderr=True, ignore_quiet=ignore_quiet) assert ("WARN: needed" in capsys.readouterr().err) is ignore_quiet + + +# ── One Transport ──────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("base", "rest", "ws"), + [ + ( + "https://api.deepgram.com", + "https://api.deepgram.com", + "wss://api.deepgram.com", + ), + ("http://127.0.0.1:8080", "http://127.0.0.1:8080", "ws://127.0.0.1:8080"), + ("ws://127.0.0.1:8080/x", "http://127.0.0.1:8080/x", "ws://127.0.0.1:8080/x"), + ("wss://h.example:9", "https://h.example:9", "wss://h.example:9"), + ("h.example", "https://h.example", "wss://h.example"), + ], +) +def test_base_derivation(base, rest, ws): + assert transport.rest_url(base) == rest + assert transport.ws_url(base) == ws + assert transport.ws_url(transport.rest_url(base)) == ws + + +def test_transport_bases_come_from_the_host_check(monkeypatch): + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080/") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + assert t.rest_base() == "http://127.0.0.1:8080" + assert t.ws_base() == "ws://127.0.0.1:8080" + assert t.auth_headers(KEY) == transport.request_headers(KEY) + assert t.auth_headers(KEY)["Authorization"] == f"Token {KEY}" + assert t.client is transport.shared_client() + + +def test_request_headers_are_the_sdk_clients_headers(monkeypatch): + """dg's own requests and the SDK's carry the same User-Agent, X-Fern + headers, session ID and Authorization: one run, one session ID.""" + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080/") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + sdk = t.sdk_client(KEY) + assert sdk._client_wrapper.get_headers() == t.auth_headers(KEY) + sid = t.auth_headers(KEY)[transport.SESSION_HEADER] + assert sid == transport.session_id() == sdk.session_id + assert len(sid) == 36 + assert t.auth_headers("sk-other")[transport.SESSION_HEADER] == sid + + +def test_no_header_for_a_refused_host(monkeypatch): + monkeypatch.setattr(output, "is_ai_agent", lambda: True) + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080") + t = transport.Transport(Config(config_path=str(Path.cwd() / "none.yaml"))) + with pytest.raises(hosts.CredentialHostError): + t.auth_headers(KEY) + with pytest.raises(hosts.CredentialHostError): + t.sdk_client(KEY) + + +def test_default_host_is_hosts_constant(): + assert transport.default_host() == "api.deepgram.com" + assert f"https://{transport.default_host()}" == hosts.DEFAULT_BASE_URL + + +def test_backend_seam(monkeypatch): + calls = [] + + class Fake: + name = "fake" + + def sdk_client(self, api_key, rest_base, ws_base): + calls.append((api_key, rest_base, ws_base)) + return "sdk" + + monkeypatch.setenv("DEEPGRAM_BASE_URL", "http://127.0.0.1:8080") + config = Config(config_path=str(Path.cwd() / "none.yaml")) + client = DeepgramClient(config, AuthManager(config)) + assert isinstance(client.transport.backend, transport.RestBackend) + client.transport.backend = Fake() + assert client.client == "sdk" + assert calls == [(KEY, "http://127.0.0.1:8080", "ws://127.0.0.1:8080")] + + +@pytest.mark.parametrize("code", [301, 302, 303, 307, 308]) +@pytest.mark.parametrize( + "call", + [ + lambda c: c.list_models(), + lambda c: c.get_projects(), + ], +) +def test_redirects_are_not_followed(code, call): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + code, headers={"location": "https://sink.example/v1/projects"} + ) + + client, seen = facade(handler) + with pytest.raises(TransportError) as err: + call(client) + assert [r.url.host for r in seen] == ["api.deepgram.com"] + assert err.value.code == "network_unavailable" + assert err.value.messages == transport.NETWORK_REDIRECT + assert err.value.details == { + "host": "api.deepgram.com", + "reason": "unexpected redirect", + } + assert err.value.error == ( + "https://api.deepgram.com answered with a redirect to " + "https://sink.example. dg doesn't follow redirects, so nothing was " + "sent there." + ) + + +@pytest.mark.parametrize( + ("agentic", "hard", "hint"), + [ + ( + False, + False, + "A network sign-in page or proxy may be redirecting requests. If " + "api.deepgram.com is your own server, use the address it redirects to.", + ), + ( + True, + True, + "Tell the user api.deepgram.com redirected the request. Don't retry " + "until they have checked the network or the configured URL.", + ), + ], +) +def test_redirect_hints(monkeypatch, agentic, hard, hint): + err = mapped(status(302, headers={"location": "https://sink.example/"})) + payload = _render(err, monkeypatch, agentic=agentic, hard=hard) + assert payload["hint"] == hint + assert payload["next_step"] is None + + +def test_redirect_on_upload_sends_the_body_once(tmp_path): + audio = tmp_path / "a.wav" + audio.write_bytes(b"RIFF" + b"\0" * 64) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(307, headers={"location": "https://sink.example/x"}) + + client, seen = facade(handler) + with pytest.raises(TransportError) as err: + client.transcribe_file(audio) + assert len(seen) == 1 and seen[0].url.path == "/v1/listen" + assert err.value.details["reason"] == "unexpected redirect" + + +def test_shared_client_follows_no_redirects(monkeypatch): + created = {} + + class Recorder(httpx.Client): + def __init__(self, **kwargs): + created.update(kwargs) + super().__init__(**kwargs) + + monkeypatch.setattr(httpx, "Client", Recorder) + monkeypatch.delenv("DEEPGRAM_CLI_INSECURE", raising=False) + client = transport.shared_client() + assert created["follow_redirects"] is False + assert created.get("verify", True) is True + assert "mounts" not in created + assert client.follow_redirects is False + + +@pytest.mark.parametrize(("agent", "loopback_mounts"), [(False, True), (True, False)]) +def test_insecure_flag_only_for_a_person(monkeypatch, capsys, agent, loopback_mounts): + created = {} + + class Recorder(httpx.Client): + def __init__(self, **kwargs): + created.update(kwargs) + super().__init__(**kwargs) + + monkeypatch.setattr(httpx, "Client", Recorder) + monkeypatch.setenv("DEEPGRAM_CLI_INSECURE", "1") + monkeypatch.setattr(output, "is_ai_agent", lambda: agent) + transport.shared_client() + # Verification stays on; a person gets unverified loopback mounts only + # (test_auth_insecure.py checks hosts and dg api's verified client). + assert created.get("verify", True) is True + assert ("mounts" in created) is loopback_mounts + err = capsys.readouterr().err + assert ("Ignoring DEEPGRAM_CLI_INSECURE" in err) is agent + + +def test_cookies_are_never_sent_back(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"models": []} if "models" in request.url.path else {"projects": []}, + headers={"set-cookie": "sid=s3cr3t; Path=/; Domain=api.deepgram.com"}, + ) + + client, seen = facade(handler) + client.list_models() + client.get_projects() + client.list_models() + assert len(seen) == 3 + assert all("cookie" not in r.headers for r in seen) + assert len(transport.shared_client().cookies.jar) == 0 + + +def test_no_code_sets_cookies_on_the_shared_client(): + """httpx copies the client's jar into each request, so a + cookie set in code would be sent anyway. Nothing may set one.""" + import re + + root = Path(__file__).resolve().parents[3] # packages/ + offenders = [] + files = list(root.glob("*/src/**/*.py")) + assert len(files) > 50 + for path in files: + text = path.read_text(encoding="utf-8") + for number, line in enumerate(text.splitlines(), 1): + if re.search(r"\.cookies\.set\(|\bcookies\s*=", line): + offenders.append(f"{path.relative_to(root)}:{number}: {line.strip()}") + assert offenders == [] + + +# ── WebSocket handshake mapping ────────────────────────────────────── + + +def _ws_refused(code: int, headers: dict | None = None) -> BaseException: + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatus + from websockets.http11 import Response + + return InvalidStatus(Response(code, "x", Headers(headers or {}), b"{}")) + + +@pytest.mark.parametrize( + ("code", "want"), + [ + (401, "auth_invalid"), + (403, "auth_forbidden"), + (429, "rate_limited"), + (400, "invalid_argument"), + (500, "server_error"), + (503, "server_error"), + (302, "network_unavailable"), + ], +) +def test_ws_refused_upgrade_maps_by_status(code, want): + exc = _ws_refused(code, {"retry-after": "7", "dg-request-id": "r-1"}) + mapped = transport.map_exception(exc) + assert mapped is not None and mapped.code == want + assert transport.map_ws_open(exc, host="api.deepgram.com").code == want + if code == 429: + assert mapped.details["retry_after"] == 7 + if code == 302: + assert mapped.details["reason"] == "unexpected redirect" + assert mapped.messages == transport.NETWORK_REDIRECT + + +def test_ws_legacy_invalid_status_code(): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatusCode + + exc = InvalidStatusCode(401, Headers()) + assert transport.map_exception(exc).code == "auth_invalid" + + +@pytest.mark.parametrize( + ("inner", "code", "reason"), + [ + ( + socket.gaierror(-2, "Name or service not known"), + "network_unavailable", + "name resolution failed", + ), + ( + ConnectionRefusedError(111, "refused"), + "network_unavailable", + "connection refused", + ), + ( + ssl.SSLError(1, "CERTIFICATE_VERIFY_FAILED"), + "network_unavailable", + "TLS handshake failed", + ), + (TimeoutError(), "timeout", None), + ], +) +def test_ws_open_failures(inner, code, reason): + try: + try: + raise inner + except BaseException as cause: + raise RuntimeError("open failed") from cause + except RuntimeError as exc: + mapped = transport.map_ws_open(exc, host="api.deepgram.com") + assert mapped.code == code + assert mapped.details["host"] == "api.deepgram.com" + if reason: + assert mapped.details["reason"] == reason + else: + assert mapped.details["timeout_s"] == transport._seconds( + transport.WS_OPEN_TIMEOUT_S + ) + + +def test_unrelated_errors_are_not_ws_errors(): + assert transport.map_exception(ValueError("x")) is None + assert transport.map_ws_open(ValueError("x"), host="h") is None + + +def test_module_aliases_kept_for_w24_w25(): + assert callable(transport.shared_client) + assert callable(transport.map_exception) + assert callable(transport.raise_for) diff --git a/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py b/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py new file mode 100644 index 00000000..17dd21bf --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_ws_handshake_errors.py @@ -0,0 +1,104 @@ +"""A WebSocket upgrade answered with something that isn't valid HTTP, and +errors the client wraps for the SDK: plain words, no field dumps.""" + +from __future__ import annotations + +import pytest +from deepctl_core import transport + +websockets_exceptions = pytest.importorskip("websockets.exceptions") + + +def _wrapped(cause: BaseException) -> BaseException: + from deepgram.core.api_error import ApiError + + try: + try: + raise cause + except Exception as e: + raise ApiError(body=f"Text-to-speech streaming failed: {e}") from None + except ApiError as wrapped: + wrapped.__context__ = cause + return wrapped + + +def test_invalid_http_answer_maps_to_unreadable(): + exc = websockets_exceptions.InvalidMessage( + "did not receive a valid HTTP response" + ) + exc.__cause__ = EOFError("connection closed while reading HTTP status line") + mapped = transport.map_exception(exc) + assert isinstance(mapped, transport.TransportError) + assert mapped.code == "network_unavailable" + assert mapped.details["reason"] == transport.REASON_UNREADABLE + + +def test_invalid_header_maps_too(): + mapped = transport.map_ws_open( + websockets_exceptions.InvalidHeader("Upgrade"), host="api.deepgram.com" + ) + assert mapped is not None + assert mapped.code == "network_unavailable" + assert mapped.details == { + "host": "api.deepgram.com", + "reason": transport.REASON_UNREADABLE, + } + + +def test_refused_status_still_maps_by_status(): + from websockets.datastructures import Headers + from websockets.http11 import Response + + response = Response(401, "Unauthorized", Headers(), b"") + mapped = transport.map_exception(websockets_exceptions.InvalidStatus(response)) + assert mapped is not None + assert mapped.code == "auth_invalid" + + +def test_plain_detail_skips_the_sdk_field_dump(): + wrapped = _wrapped(RuntimeError("socket closed")) + assert "status_code" in str(wrapped) # the dump this avoids + assert transport.plain_detail(wrapped) == "socket closed" + + +def test_plain_detail_keeps_an_ordinary_message(): + assert transport.plain_detail(ValueError("bad value")) == "bad value" + assert transport.plain_detail(ValueError()) == "ValueError" + + +def test_plain_detail_keeps_an_api_error_with_a_status(): + from deepgram.core.api_error import ApiError + + err = ApiError(status_code=500, body="oops") + assert "500" in transport.plain_detail(err) + + +def test_rest_redirect_names_the_running_command(): + import click + + @click.group(name="dg") + def root() -> None: + pass + + @root.command(name="models") + def models() -> None: + err = transport.for_status( + 307, {"location": "https://elsewhere.example/x"}, host="api.deepgram.com" + ) + assert err is not None + click.echo(err.message) + + from click.testing import CliRunner + + result = CliRunner().invoke(root, ["models"]) + assert "dg models doesn't follow redirects, so nothing was sent there." in ( + " ".join(result.output.split()) + ), result.output + + +def test_rest_redirect_outside_a_command_says_dg(): + err = transport.for_status( + 307, {"location": "https://elsewhere.example/x"}, host="api.deepgram.com" + ) + assert err is not None + assert "dg doesn't follow redirects" in err.message diff --git a/src/deepctl/_cli.py b/src/deepctl/_cli.py index 735e19ab..d547f247 100644 --- a/src/deepctl/_cli.py +++ b/src/deepctl/_cli.py @@ -826,6 +826,12 @@ def cli( # came from the command line, so it is never saved and gets # credentials only if the host is trusted. ctx.ensure_object(dict) + # An unexpanded ${...} in a DEEPGRAM_* variable counts as unset + # (W-ENV-PLACEHOLDER, printed once the output mode is known). + from deepctl_core.credentials import drop_env_placeholders + + drop_env_placeholders() + from deepctl_core import Config # Help from a spec runs no command: only the files' checks and diff --git a/src/deepctl/_manifest.d/update.json b/src/deepctl/_manifest.d/update.json index 5e98d438..f292ae7a 100644 --- a/src/deepctl/_manifest.d/update.json +++ b/src/deepctl/_manifest.d/update.json @@ -8,7 +8,7 @@ "requires_auth": false, "help": { "name": "update", - "help": "Check for and install updates to deepctl\n\nExamples:\n dg update\n dg update --check-only\n\n\nFor agents, use --agent-friendly.", + "help": "Check for and install updates to deepctl\n\nExamples:\n dg update\n dg update --check-only\n dg update --dry-run\n\n\nFor agents, use --agent-friendly.", "short_help": "Check for and install updates to deepctl", "epilog": null, "hidden": false, @@ -59,6 +59,27 @@ "hidden": false, "help": "Force update even if already up to date" }, + { + "class": "option", + "name": "dry_run", + "opts": [ + "--dry-run" + ], + "secondary_opts": [], + "type": { + "name": "boolean" + }, + "required": false, + "nargs": 1, + "metavar": null, + "eager": false, + "expose": true, + "multiple": false, + "is_flag": true, + "count": false, + "hidden": false, + "help": "Show how deepctl was installed and the command that updates it, without checking for or installing updates" + }, { "class": "option", "name": "check_only", diff --git a/tests/unit/test_base_url_persistence.py b/tests/unit/test_base_url_persistence.py index 88f00da6..3cd0479a 100644 --- a/tests/unit/test_base_url_persistence.py +++ b/tests/unit/test_base_url_persistence.py @@ -90,14 +90,6 @@ def _ffprobe_reset(config, auth, client): FfprobeCommand().handle(config, auth, client, reset=True) -def _version_cache(config, auth, client): - VersionChecker(config)._cache_info( - VersionInfo( - current_version="0.4.0", latest_version="0.4.1", update_available=True - ) - ) - - def _device_login(config, auth, client): auth._store_token( TokenResponse( @@ -124,7 +116,6 @@ def _delete_and_create(config, auth, client): _switch_profile, _ffprobe_set, _ffprobe_reset, - _version_cache, _device_login, _update_install_method, _delete_and_create, @@ -171,3 +162,21 @@ def test_command_never_saves_runtime_base_url(tmp_path, monkeypatch, saver, via) monkeypatch.delenv("DEEPGRAM_BASE_URL", raising=False) later = Config(config_path=str(path)) assert later.get_profile().base_url == "https://api.deepgram.com" + + +def test_version_cache_never_writes_the_config(tmp_path, monkeypatch): + """The update check's answer goes to the cache directory instead.""" + from deepctl import _cache + + monkeypatch.setattr(_cache, "cache_root", lambda: str(tmp_path / "cache")) + path = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump({"profiles": {"default": {"project_id": "P1"}}})) + before = path.read_bytes() + config = Config(config_path=str(path), base_url=URL) + VersionChecker(config)._cache_info( + VersionInfo( + current_version="0.4.0", latest_version="0.4.1", update_available=True + ) + ) + assert path.read_bytes() == before + assert (tmp_path / "cache" / "update-check.json").is_file() diff --git a/tests/unit/test_command_imports.py b/tests/unit/test_command_imports.py index cbc82828..96c492a3 100644 --- a/tests/unit/test_command_imports.py +++ b/tests/unit/test_command_imports.py @@ -91,19 +91,7 @@ def _is_group(path: str) -> bool: @pytest.mark.parametrize( "path", [ - pytest.param( - p.split(" "), - marks=pytest.mark.xfail( - strict=True, - reason=( - "a subcommand group's help takes the full path, where the " - "parent group's callback builds an API client (httpx, ssl, " - "keyring) before the subcommand parses --help" - ), - ) - if _is_group(p) - else (), - ) + p.split(" ") for p in sorted(_manifest.commands()) if " " in p ],