From dbd3a3324d25ad85b89aef7f2350b16df046bb81 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 30 Oct 2026 16:25:12 -0400 Subject: [PATCH 01/15] feat(cli): detect npm launcher installs in update, plugin and skills The @deepgram/cli npm package runs deepctl with DEEPCTL_LAUNCHER set. deepctl-core gains deepctl_core.launcher, which reads that variable once and drops it from the environment. deepctl-cmd-update reports the npm install method and gives npm-shaped update advice. deepctl-cmd-plugin installs plugins into its isolated venv, because the launcher's venv is replaced on every npm version bump. deepctl-cmd-skills shows its one-time prompt even when the launcher runs deepctl through uv. Those packages now need the new core API, so their floors move to deepctl-core>=0.3.0, and plugin declares deepctl-cmd-update>=0.2.7 (with a workspace source) for InstallMethod.NPM. uv.lock is regenerated. --- conftest.py | 10 +++ packages/deepctl-cmd-plugin/pyproject.toml | 5 ++ .../src/deepctl_cmd_plugin/command.py | 7 +- .../src/deepctl_cmd_plugin/strategies.py | 2 +- .../tests/unit/test_plugin_command.py | 1 + .../tests/unit/test_strategies.py | 10 +++ packages/deepctl-cmd-skills/pyproject.toml | 2 +- .../src/deepctl_cmd_skills/startup_check.py | 8 +- .../tests/unit/test_skills_command.py | 32 +++++++ packages/deepctl-cmd-update/pyproject.toml | 2 +- .../src/deepctl_cmd_update/installation.py | 26 +++++- .../deepctl_cmd_update/plugin_update_check.py | 4 +- .../src/deepctl_cmd_update/startup_check.py | 16 +++- .../src/deepctl_cmd_update/version_check.py | 4 +- .../tests/unit/test_installation.py | 85 ++++++++++++++++++- .../tests/unit/test_plugin_update_check.py | 4 +- .../tests/unit/test_startup_check.py | 44 ++++++++++ .../tests/unit/test_version_check.py | 11 +++ .../deepctl-core/src/deepctl_core/launcher.py | 53 ++++++++++++ .../deepctl-core/tests/unit/test_launcher.py | 45 ++++++++++ src/deepctl/main.py | 5 ++ tests/unit/test_main.py | 22 +++++ uv.lock | 2 + 23 files changed, 386 insertions(+), 14 deletions(-) create mode 100644 packages/deepctl-core/src/deepctl_core/launcher.py create mode 100644 packages/deepctl-core/tests/unit/test_launcher.py diff --git a/conftest.py b/conftest.py index fc8e6cb2..89d1f5c3 100644 --- a/conftest.py +++ b/conftest.py @@ -16,3 +16,13 @@ def _clean_deepgram_env(monkeypatch): for key in list(os.environ): if key.startswith("DEEPGRAM_"): monkeypatch.delenv(key, raising=False) + + +@pytest.fixture(autouse=True) +def _fresh_launcher_capture(): + """Re-read DEEPCTL_LAUNCHER in each test, so env patches take effect.""" + from deepctl_core.launcher import reset_launcher_for_tests + + reset_launcher_for_tests() + yield + reset_launcher_for_tests() diff --git a/packages/deepctl-cmd-plugin/pyproject.toml b/packages/deepctl-cmd-plugin/pyproject.toml index adf96e64..8f6ab191 100644 --- a/packages/deepctl-cmd-plugin/pyproject.toml +++ b/packages/deepctl-cmd-plugin/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.10" authors = [{ name = "Deepgram DevRel", email = "devrel@deepgram.com" }] dependencies = [ "deepctl-core>=0.2.19", + "deepctl-cmd-update>=0.2.7", "packaging>=21.0", "toml>=0.10.2", ] @@ -14,6 +15,10 @@ dependencies = [ [project.entry-points."deepctl.commands"] plugin = "deepctl_cmd_plugin.command:PluginCommand" +[tool.uv.sources] +deepctl-core = { workspace = true } +deepctl-cmd-update = { workspace = true } + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py index dd14fa5a..a5228d57 100644 --- a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py +++ b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py @@ -694,11 +694,16 @@ def _get_plugin_registry(self) -> list[PluginRegistryEntry]: ] def _needs_isolated_venv(self, method: InstallMethod) -> bool: - """Check if this installation method needs the isolated plugin venv.""" + """Check if this installation method needs the isolated plugin venv. + + npm counts: the launcher's venv or uv environment is replaced on every + npm version bump, so plugins installed into it would vanish. + """ return method in ( InstallMethod.HOMEBREW, InstallMethod.SYSTEM, InstallMethod.UNKNOWN, + InstallMethod.NPM, ) def _get_strategy( diff --git a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/strategies.py b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/strategies.py index 90a36289..e8dc9120 100644 --- a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/strategies.py +++ b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/strategies.py @@ -272,7 +272,7 @@ def get_strategy( if method in (InstallMethod.UVX, InstallMethod.PIPX_RUN): return EphemeralStrategy(method) - # HOMEBREW, SYSTEM, UNKNOWN → isolated venv + # HOMEBREW, SYSTEM, UNKNOWN, NPM → isolated venv if venv_python is None: raise ValueError( f"venv_python is required for IsolatedVenvStrategy (method={method.value})" diff --git a/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py b/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py index b9f8a692..c9343c83 100644 --- a/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py +++ b/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py @@ -768,6 +768,7 @@ def test_needs_isolated_venv(self) -> None: assert self.command._needs_isolated_venv(InstallMethod.HOMEBREW) is True assert self.command._needs_isolated_venv(InstallMethod.SYSTEM) is True assert self.command._needs_isolated_venv(InstallMethod.UNKNOWN) is True + assert self.command._needs_isolated_venv(InstallMethod.NPM) is True assert self.command._needs_isolated_venv(InstallMethod.PIP) is False assert self.command._needs_isolated_venv(InstallMethod.PIPX) is False assert self.command._needs_isolated_venv(InstallMethod.UV) is False diff --git a/packages/deepctl-cmd-plugin/tests/unit/test_strategies.py b/packages/deepctl-cmd-plugin/tests/unit/test_strategies.py index f9217cfd..f43b4863 100644 --- a/packages/deepctl-cmd-plugin/tests/unit/test_strategies.py +++ b/packages/deepctl-cmd-plugin/tests/unit/test_strategies.py @@ -313,6 +313,16 @@ def test_homebrew_returns_isolated_venv(self) -> None: ) assert isinstance(strategy, IsolatedVenvStrategy) + @pytest.mark.unit + def test_npm_returns_isolated_venv(self) -> None: + strategy = get_strategy(InstallMethod.NPM, venv_python="/path/to/python") + assert isinstance(strategy, IsolatedVenvStrategy) + + @pytest.mark.unit + def test_npm_without_venv_python_raises(self) -> None: + with pytest.raises(ValueError, match="venv_python is required"): + get_strategy(InstallMethod.NPM) + @pytest.mark.unit def test_unknown_returns_isolated_venv(self) -> None: strategy = get_strategy( diff --git a/packages/deepctl-cmd-skills/pyproject.toml b/packages/deepctl-cmd-skills/pyproject.toml index 8e9bd3a6..39030aca 100644 --- a/packages/deepctl-cmd-skills/pyproject.toml +++ b/packages/deepctl-cmd-skills/pyproject.toml @@ -6,7 +6,7 @@ version = "0.0.9" # x-release-please-version requires-python = ">=3.10" authors = [{ name = "Deepgram DevRel", email = "devrel@deepgram.com" }] dependencies = [ - "deepctl-core>=0.2.19", + "deepctl-core>=0.3.0", "click>=8.0.0", "rich>=13.0.0", ] diff --git a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py index 8d153221..ed6ae711 100644 --- a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py +++ b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py @@ -11,6 +11,8 @@ import threading from pathlib import Path +from deepctl_core.launcher import launched_via_npm + # Environment variables that indicate a CI environment _CI_ENV_VARS = ( "CI", @@ -33,6 +35,10 @@ def _is_ci() -> bool: def _is_oneshot() -> bool: + # The npm launcher may run deepctl via `uv tool run`, but its install + # persists, so a one-time prompt is worth showing there too. + if launched_via_npm(): + return False if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: return True exe_path = str(Path(sys.executable).resolve()) @@ -110,7 +116,7 @@ def print_pending_notification() -> None: try: sys.stderr.write( "\033[36mAI coding assistants detected. " - "Run 'deepctl skills install' to set up integration.\033[0m\n" + "Run 'dg skills install' to set up integration.\033[0m\n" ) except (BrokenPipeError, OSError, ValueError): # The output stream closed (e.g. an MCP host disconnected stdio diff --git a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py index 700bc305..9717c0b2 100644 --- a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py +++ b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py @@ -965,6 +965,38 @@ def test_suppressed_when_quiet(self): startup_check.check_and_notify(quiet=True) assert startup_check._thread is None + def test_npm_launcher_is_not_oneshot(self): + """npm runs deepctl via uv, but the npm install persists.""" + from deepctl_cmd_skills import startup_check + + env = { + "UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3", + "DEEPCTL_LAUNCHER": "npm", + } + with patch.dict(os.environ, env, clear=False): + assert startup_check._is_oneshot() is False + from deepctl_core.launcher import reset_launcher_for_tests + + reset_launcher_for_tests() + with patch.dict(os.environ, env, clear=False): + del os.environ["DEEPCTL_LAUNCHER"] + assert startup_check._is_oneshot() is True + + def test_prompt_names_dg(self): + """npm installs only `dg` and `deepgram`, so the prompt must not say deepctl.""" + import io + import threading + + from deepctl_cmd_skills import startup_check + + startup_check._result = {"should_prompt": True} + startup_check._thread = threading.Thread(target=lambda: None) + startup_check._thread.start() + err = io.StringIO() + with patch.object(sys, "stderr", err): + startup_check.print_pending_notification() + assert "Run 'dg skills install'" in err.getvalue() + @pytest.mark.parametrize( "error", [ diff --git a/packages/deepctl-cmd-update/pyproject.toml b/packages/deepctl-cmd-update/pyproject.toml index cb06ffd1..77d9a300 100644 --- a/packages/deepctl-cmd-update/pyproject.toml +++ b/packages/deepctl-cmd-update/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ keywords = ["deepgram", "cli", "update", "self-update"] requires-python = ">=3.10" dependencies = [ - "deepctl-core>=0.1.10", + "deepctl-core>=0.3.0", "click>=8.0.0", "pydantic>=2.0.0", "rich>=13.0.0", diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py index 873df081..c75e864d 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py @@ -7,6 +7,7 @@ from enum import Enum from pathlib import Path +from deepctl_core.launcher import get_launcher, launched_via_npm from pydantic import BaseModel @@ -20,6 +21,7 @@ class InstallMethod(str, Enum): UV_TOOL = "uv_tool" UVX = "uvx" PIPX_RUN = "pipx_run" + NPM = "npm" SYSTEM = "system" DEVELOPMENT = "development" UNKNOWN = "unknown" @@ -105,6 +107,7 @@ def get_update_command(self, method: InstallMethod) -> list[str] | None: ], InstallMethod.UVX: None, # Ephemeral, no update needed InstallMethod.PIPX_RUN: None, # Ephemeral, no update needed + InstallMethod.NPM: None, # Updated through npm, see instructions InstallMethod.SYSTEM: None, # Handled separately InstallMethod.DEVELOPMENT: None, # Can't auto-update InstallMethod.UNKNOWN: None, @@ -123,6 +126,20 @@ def get_update_instructions(self, info: InstallationInfo) -> str: if info.method == InstallMethod.HOMEBREW: return "Please update using Homebrew: brew upgrade deepctl" + # The launcher's venv or uv cache is private to it, so pip/uv run + # from inside would update the wrong thing; only npm can update it. + if info.method == InstallMethod.NPM: + if get_launcher() == "npx": + return ( + "Running via npx. Use the latest version with: " + "npx @deepgram/cli@latest" + ) + return ( + "Installed via npm. Update with: " + "npm install -g @deepgram/cli@latest " + "(or run npx @deepgram/cli@latest)" + ) + if info.method == InstallMethod.UVX: return ( "You are running deepctl via uvx (one-shot execution). " @@ -261,8 +278,8 @@ def _detect_method( ) -> InstallMethod: """Detect the installation method. - Priority: development > oneshot > homebrew > uv_tool > pipx > - uv > system > pip > unknown + Priority: npm > development > oneshot > homebrew > uv_tool > + pipx > uv > system > pip > unknown Args: package_location: Path to package @@ -272,6 +289,11 @@ def _detect_method( Returns: Detected installation method """ + # The npm launcher runs deepctl through uv or its own venv, which + # would otherwise be detected as uvx or pip. + if launched_via_npm(): + return InstallMethod.NPM + # Development installation if is_editable: return InstallMethod.DEVELOPMENT diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py index e490cb3b..00517ada 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/plugin_update_check.py @@ -261,7 +261,7 @@ def print_pending_plugin_notifications() -> None: u = updates[0] sys.stderr.write( f"\033[33mPlugin update available: {u['name']} {u['current']} → {u['latest']}" - f" — run 'deepctl plugin update {u['name']}' to upgrade\033[0m\n" + f" — run 'dg plugin update {u['name']}' to upgrade\033[0m\n" ) else: sys.stderr.write("\033[33mPlugin updates available:\033[0m\n") @@ -270,7 +270,7 @@ def print_pending_plugin_notifications() -> None: f"\033[33m {u['name']} {u['current']} → {u['latest']}\033[0m\n" ) sys.stderr.write( - "\033[33mRun 'deepctl plugin update ' to upgrade\033[0m\n" + "\033[33mRun 'dg plugin update ' to upgrade\033[0m\n" ) except (BrokenPipeError, OSError, ValueError): # The output stream closed (e.g. an MCP host disconnected stdio diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py index 889bab17..bbaec963 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py @@ -15,6 +15,8 @@ from pathlib import Path from typing import Any +from deepctl_core.launcher import launched_via_npm, npm_upgrade_command + # Sentinel used when the background check is suppressed or has no result _NO_RESULT: dict[str, Any] = {} @@ -42,8 +44,20 @@ def _is_ci() -> bool: return any(os.environ.get(v) for v in _CI_ENV_VARS) +def upgrade_hint() -> str: + """Return the command a user should run to upgrade deepctl.""" + if launched_via_npm(): + return npm_upgrade_command() + return "dg update" + + def _is_oneshot() -> bool: """Return True if running in an ephemeral one-shot environment.""" + # The npm launcher often runs deepctl through `uv tool run`, which looks + # one-shot, but the npm install itself persists and is pinned to one + # version, so its users do need to hear about updates. + if launched_via_npm(): + return False # uvx if "UV_INTERNAL__PARENT_INTERPRETER" in os.environ: return True @@ -200,7 +214,7 @@ def print_pending_notification() -> None: try: sys.stderr.write( f"\n\033[33m Update available: {current} → {latest}" - f" — run \033[1mdg update\033[0m\033[33m to upgrade\033[0m\n\n" + f" — run \033[1m{upgrade_hint()}\033[0m\033[33m to upgrade\033[0m\n\n" ) except (BrokenPipeError, OSError, ValueError): # The output stream closed (e.g. an MCP host disconnected stdio diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py index 89a1fa43..33384823 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/version_check.py @@ -7,6 +7,8 @@ from packaging import version from pydantic import BaseModel, Field +from .startup_check import upgrade_hint + class VersionInfo(BaseModel): """Version information from PyPI.""" @@ -270,5 +272,5 @@ def format_version_message(info: VersionInfo) -> str: else: message += f" (released {days_old} days ago)" - message += "\nRun 'dg update' to upgrade" + message += f"\nRun '{upgrade_hint()}' to upgrade" return message diff --git a/packages/deepctl-cmd-update/tests/unit/test_installation.py b/packages/deepctl-cmd-update/tests/unit/test_installation.py index b98cf306..c932dfca 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_installation.py +++ b/packages/deepctl-cmd-update/tests/unit/test_installation.py @@ -22,7 +22,7 @@ def test_all_methods_defined(self): """Verify all expected methods exist.""" expected = { "pip", "pipx", "uv", "homebrew", "uv_tool", - "uvx", "pipx_run", "system", "development", "unknown", + "uvx", "pipx_run", "npm", "system", "development", "unknown", } actual = {m.value for m in InstallMethod} assert actual == expected @@ -68,6 +68,9 @@ def test_returns_none_for_pipx_run(self, detector): cmd = detector.get_update_command(InstallMethod.PIPX_RUN) assert cmd is None + def test_returns_none_for_npm(self, detector): + assert detector.get_update_command(InstallMethod.NPM) is None + def test_returns_none_for_development(self, detector): cmd = detector.get_update_command(InstallMethod.DEVELOPMENT) assert cmd is None @@ -263,3 +266,83 @@ def test_pipx_run_instructions(self, detector): instructions = detector.get_update_instructions(info) assert "pipx run" in instructions assert "automatically" in instructions + + def test_npm_instructions(self, detector): + from deepctl_cmd_update.installation import InstallationInfo + + info = InstallationInfo( + method=InstallMethod.NPM, + path="/home/u/.cache/deepgram-cli/venvs/0.4.0", + virtual_env=True, + editable=False, + python_executable="/home/u/.cache/deepgram-cli/venvs/0.4.0/bin/python", + ) + instructions = detector.get_update_instructions(info) + assert "npm install -g @deepgram/cli@latest" in instructions + assert "npx @deepgram/cli@latest" in instructions + + + @patch.dict("os.environ", {"DEEPCTL_LAUNCHER": "npx"}, clear=False) + def test_npx_instructions(self, detector): + from deepctl_cmd_update.installation import InstallationInfo + + info = InstallationInfo( + method=InstallMethod.NPM, + path="/home/u/.cache/deepgram-cli/venvs/0.4.0", + virtual_env=True, + editable=False, + python_executable="/home/u/.cache/deepgram-cli/venvs/0.4.0/bin/python", + ) + instructions = detector.get_update_instructions(info) + assert "npx @deepgram/cli@latest" in instructions + assert "npm install -g" not in instructions + + +class TestNpmLauncherDetection: + """The npm launcher marks deepctl's environment with DEEPCTL_LAUNCHER=npm.""" + + @patch.dict( + "os.environ", + { + "DEEPCTL_LAUNCHER": "npm", + # The launcher's uv strategy also looks like uvx; npm must win. + "UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3", + }, + clear=False, + ) + def test_npm_wins_over_uvx_and_development(self): + detector = InstallationDetector() + assert ( + detector._detect_method(None, in_venv=True, is_editable=True) + == InstallMethod.NPM + ) + + @patch.dict( + "os.environ", + {"UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3"}, + clear=False, + ) + def test_other_launcher_values_are_ignored(self): + import os + + os.environ["DEEPCTL_LAUNCHER"] = "something-else" + detector = InstallationDetector() + assert ( + detector._detect_method(None, in_venv=False, is_editable=False) + == InstallMethod.UVX + ) + + @patch.dict("os.environ", {"DEEPCTL_LAUNCHER": "npx"}, clear=False) + def test_npx_is_the_npm_method(self): + detector = InstallationDetector() + assert ( + detector._detect_method(None, in_venv=True, is_editable=False) + == InstallMethod.NPM + ) + + @patch.dict("os.environ", {}, clear=False) + def test_detect_reports_npm(self): + import os + + os.environ["DEEPCTL_LAUNCHER"] = "npm" + assert InstallationDetector().detect().method == InstallMethod.NPM diff --git a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py index 9de3b700..a4f9f0a0 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_plugin_update_check.py @@ -365,7 +365,7 @@ def test_single_plugin_format(self): assert "deepctl-plugin-whisper" in output assert "0.1.0" in output assert "0.2.0" in output - assert "deepctl plugin update deepctl-plugin-whisper" in output + assert "dg plugin update deepctl-plugin-whisper" in output def test_multiple_plugin_format(self): import deepctl_cmd_update.plugin_update_check as mod @@ -392,7 +392,7 @@ def test_multiple_plugin_format(self): assert "Plugin updates available:" in output assert "deepctl-plugin-whisper" in output assert "deepctl-plugin-asr" in output - assert "deepctl plugin update " in output + assert "dg plugin update " in output @pytest.mark.parametrize( "error", diff --git a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py index 8d3962c5..cc26c848 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_startup_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_startup_check.py @@ -66,6 +66,30 @@ def test_not_oneshot(self): def test_uvx_oneshot(self): assert _is_oneshot() is True + @patch.dict( + "os.environ", + { + "UV_INTERNAL__PARENT_INTERPRETER": "/usr/bin/python3", + "DEEPCTL_LAUNCHER": "npm", + }, + clear=True, + ) + def test_npm_launcher_is_not_oneshot(self): + """npm runs deepctl via uv, but the npm install persists.""" + assert _is_oneshot() is False + + @patch.dict("os.environ", {"DEEPCTL_LAUNCHER": "npx"}, clear=True) + def test_upgrade_hint_under_npx(self): + from deepctl_cmd_update.startup_check import upgrade_hint + + assert upgrade_hint() == "npx @deepgram/cli@latest" + + @patch.dict("os.environ", {}, clear=True) + def test_upgrade_hint_default(self): + from deepctl_cmd_update.startup_check import upgrade_hint + + assert upgrade_hint() == "dg update" + class TestCacheTimestamp: """Test file-based timestamp caching.""" @@ -208,6 +232,26 @@ def test_notification_displayed(self): assert "1.0.0" in output assert "dg update" in output + @patch.dict("os.environ", {"DEEPCTL_LAUNCHER": "npm"}, clear=False) + def test_notification_names_npm_under_the_launcher(self): + """`dg update` cannot update an npm install; say what can.""" + import io + + import deepctl_cmd_update.startup_check as mod + + mod._result = {"latest": "2.0.0", "current": "1.0.0"} + mod._thread = threading.Thread(target=lambda: None) + mod._thread.start() + mod._thread.join() + + stderr_capture = io.StringIO() + with patch.object(sys, "stderr", stderr_capture): + print_pending_notification() + + output = stderr_capture.getvalue() + assert "npm install -g @deepgram/cli@latest" in output + assert "dg update" not in output + def test_no_notification_when_up_to_date(self, capsys): """No output when no update is available.""" import deepctl_cmd_update.startup_check as mod diff --git a/packages/deepctl-cmd-update/tests/unit/test_version_check.py b/packages/deepctl-cmd-update/tests/unit/test_version_check.py index f96f4fc6..35e12f5a 100644 --- a/packages/deepctl-cmd-update/tests/unit/test_version_check.py +++ b/packages/deepctl-cmd-update/tests/unit/test_version_check.py @@ -274,3 +274,14 @@ def test_update_available(self): assert "1.0.0" in msg assert "2.0.0" in msg assert "dg update" in msg + + @patch.dict("os.environ", {"DEEPCTL_LAUNCHER": "npm"}, clear=False) + def test_update_available_under_npm_launcher(self): + info = VersionInfo( + current_version="1.0.0", + latest_version="2.0.0", + update_available=True, + ) + msg = format_version_message(info) + assert "npm install -g @deepgram/cli@latest" in msg + assert "dg update" not in msg diff --git a/packages/deepctl-core/src/deepctl_core/launcher.py b/packages/deepctl-core/src/deepctl_core/launcher.py new file mode 100644 index 00000000..58b31d3c --- /dev/null +++ b/packages/deepctl-core/src/deepctl_core/launcher.py @@ -0,0 +1,53 @@ +"""Which wrapper, if any, started deepctl. + +The ``@deepgram/cli`` npm package runs deepctl from a private venv or through +``uv tool run`` and says so in ``DEEPCTL_LAUNCHER``: ``npm`` for a global +install, ``npx`` for ``npx``/``npm exec``. Nothing else sets it. + +The variable is read once and removed from ``os.environ``, so it describes +this process only. Anything deepctl spawns -- a starter app, a shell, another +``dg`` -- must not inherit a claim about how *it* was installed. +""" + +from __future__ import annotations + +import os + +LAUNCHER_ENV_VAR = "DEEPCTL_LAUNCHER" +NPM_LAUNCHERS = frozenset({"npm", "npx"}) + +_captured = False +_launcher: str | None = None + + +def capture_launcher() -> str | None: + """Read and remove ``DEEPCTL_LAUNCHER`` (first call only); return its value.""" + global _captured, _launcher + if not _captured: + _launcher = os.environ.pop(LAUNCHER_ENV_VAR, None) or None + _captured = True + return _launcher + + +def get_launcher() -> str | None: + """Return the launcher that started deepctl, or ``None``.""" + return capture_launcher() + + +def launched_via_npm() -> bool: + """Return True if the npm package (global install or npx) started deepctl.""" + return get_launcher() in NPM_LAUNCHERS + + +def npm_upgrade_command() -> str: + """Return the command that upgrades an npm-launched deepctl.""" + if get_launcher() == "npx": + return "npx @deepgram/cli@latest" + return "npm install -g @deepgram/cli@latest" + + +def reset_launcher_for_tests() -> None: + """Forget the captured value so the next call re-reads the environment.""" + global _captured, _launcher + _captured = False + _launcher = None diff --git a/packages/deepctl-core/tests/unit/test_launcher.py b/packages/deepctl-core/tests/unit/test_launcher.py new file mode 100644 index 00000000..ae954b26 --- /dev/null +++ b/packages/deepctl-core/tests/unit/test_launcher.py @@ -0,0 +1,45 @@ +"""Unit tests for deepctl_core.launcher.""" + +import os +from unittest.mock import patch + +from deepctl_core import launcher + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "npm"}, clear=False) +def test_capture_reads_once_and_removes_the_variable(): + assert launcher.capture_launcher() == "npm" + assert "DEEPCTL_LAUNCHER" not in os.environ + # A later change to the environment does not alter the captured value. + os.environ["DEEPCTL_LAUNCHER"] = "npx" + assert launcher.get_launcher() == "npm" + assert os.environ["DEEPCTL_LAUNCHER"] == "npx" + + +@patch.dict(os.environ, {}, clear=False) +def test_unset_means_no_launcher(): + os.environ.pop("DEEPCTL_LAUNCHER", None) + assert launcher.get_launcher() is None + assert launcher.launched_via_npm() is False + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": ""}, clear=False) +def test_empty_value_means_no_launcher(): + assert launcher.get_launcher() is None + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "npm"}, clear=False) +def test_npm_global_install_upgrade_command(): + assert launcher.launched_via_npm() is True + assert launcher.npm_upgrade_command() == "npm install -g @deepgram/cli@latest" + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "npx"}, clear=False) +def test_npx_upgrade_command(): + assert launcher.launched_via_npm() is True + assert launcher.npm_upgrade_command() == "npx @deepgram/cli@latest" + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "something-else"}, clear=False) +def test_unknown_launcher_is_not_npm(): + assert launcher.launched_via_npm() is False diff --git a/src/deepctl/main.py b/src/deepctl/main.py index cb8fb512..7b67a33f 100644 --- a/src/deepctl/main.py +++ b/src/deepctl/main.py @@ -19,9 +19,14 @@ print_timing_summary, setup_output, ) +from deepctl_core.launcher import capture_launcher from deepctl_core.output import stderr_console from rich.traceback import install +# Read DEEPCTL_LAUNCHER (set by the @deepgram/cli npm package) once and drop it +# from the environment, so processes deepctl starts do not inherit it. +LAUNCHER = capture_launcher() + # Install rich traceback for better error messages install(show_locals=True) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7ad33fa6..deff9351 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -336,3 +336,25 @@ def test_usage_error_without_hint_keeps_contract(self, capsys): captured = capsys.readouterr() assert captured.out == "" assert "Error" in captured.err + + +def test_launcher_marker_is_captured_and_not_inherited(): + """DEEPCTL_LAUNCHER describes this process only; children must not see it.""" + import os + import subprocess + + code = ( + "import os, subprocess, sys\n" + "import importlib\n" + "m = importlib.import_module('deepctl.main')\n" + "child = subprocess.run([sys.executable, '-c', " + "'import os; print(os.environ.get(\"DEEPCTL_LAUNCHER\"))'], " + "capture_output=True, text=True).stdout.strip()\n" + "print(m.LAUNCHER, child)\n" + ) + env = {**os.environ, "DEEPCTL_LAUNCHER": "npm"} + out = subprocess.run( + [sys.executable, "-c", code], env=env, capture_output=True, text=True + ) + assert out.returncode == 0, out.stderr + assert out.stdout.split() == ["npm", "None"] diff --git a/uv.lock b/uv.lock index fdb1750a..ec6504d7 100644 --- a/uv.lock +++ b/uv.lock @@ -1491,6 +1491,7 @@ name = "deepctl-cmd-plugin" version = "0.1.14" source = { editable = "packages/deepctl-cmd-plugin" } dependencies = [ + { name = "deepctl-cmd-update" }, { name = "deepctl-core" }, { name = "packaging" }, { name = "toml" }, @@ -1498,6 +1499,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "deepctl-cmd-update", editable = "packages/deepctl-cmd-update" }, { name = "deepctl-core", editable = "packages/deepctl-core" }, { name = "packaging", specifier = ">=21.0" }, { name = "toml", specifier = ">=0.10.2" }, From 4e8a7c02ebd6afedb819ce45f4b23a1b9e563391 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 30 Oct 2026 16:25:12 -0400 Subject: [PATCH 02/15] feat(npm): add the @deepgram/cli launcher package A zero-dependency Node launcher (Node >=20) that runs deepctl at the npm package's own version: a cached private venv, then `uv tool run --from deepctl==`, then a venv it builds under a lock, then a clear error. First-run setup is async and supervised. Signal handlers go in before setup starts, and each setup step runs in its own session. A signal kills the step, removes the partial venv, releases the lock and re-raises the signal. The lock's owner.json is always replaced atomically and records the running step's process group, so on Linux a run that breaks a stale lock can stop a pip left behind by a SIGKILLed launcher. Cached venvs are reused only from roots that pass an ownership check. Windows can't verify ACLs or reparse points, so it never reuses a cache: it runs through uv, or builds a one-run venv and deletes it afterwards. The README documents one known limitation: Node resets inherited SIG_IGN before any JS runs, so `nohup dg` and `trap '' INT` can't protect deepctl. .gitignore stops ignoring npm/lib, and .gitattributes accepts that file's CRLF line endings so `git diff --check` stays clean. --- .gitattributes | 2 + .gitignore | 2 + npm/LICENSE | 21 + npm/README.md | 91 +++ npm/bin/dg.js | 4 + npm/lib/launcher.js | 1229 +++++++++++++++++++++++++++++++++++++ npm/package.json | 42 ++ npm/test/launcher.test.js | 1224 ++++++++++++++++++++++++++++++++++++ 8 files changed, 2615 insertions(+) create mode 100644 npm/LICENSE create mode 100644 npm/README.md create mode 100755 npm/bin/dg.js create mode 100644 npm/lib/launcher.js create mode 100644 npm/package.json create mode 100644 npm/test/launcher.test.js diff --git a/.gitattributes b/.gitattributes index a86c5d21..1bceaa27 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,5 @@ # output.py is CRLF on main and ruff format keeps it that way; let git diff --check accept it. packages/deepctl-core/src/deepctl_core/output.py whitespace=cr-at-eol +# .gitignore is CRLF on main too; same treatment so new entries match the file. +.gitignore whitespace=cr-at-eol packages/deepctl-core/tests/unit/fixtures/legacy_v03/** -text diff --git a/.gitignore b/.gitignore index 77c72295..388075d8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ downloads/ eggs/ .eggs/ lib/ +# The npm launcher's sources, not a build output. +!/npm/lib/ lib64/ parts/ sdist/ diff --git a/npm/LICENSE b/npm/LICENSE new file mode 100644 index 00000000..880c2996 --- /dev/null +++ b/npm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025-present Deepgram, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 00000000..93b260c6 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,91 @@ +# @deepgram/cli + +The official [Deepgram CLI](https://github.com/deepgram/cli): speech-to-text, +text-to-speech, voice agents, audio intelligence and project management from +your terminal. + +```bash +npx @deepgram/cli --help +``` + +or install it globally: + +```bash +npm install -g @deepgram/cli +dg login +dg listen recording.wav +dg speak "Hello from Deepgram" -o hello.wav +``` + +The package installs two commands, `dg` and `deepgram`. They are the same. + +## How it works + +The Deepgram CLI is written in Python and published to PyPI as +[`deepctl`](https://pypi.org/project/deepctl/). This npm package is a small +launcher with no dependencies and no install scripts. It runs `deepctl` at the +same version as the npm package: + +1. If an earlier run already set up a private Python environment for this + version, it runs that. This is fast and works offline. +2. Otherwise, if [uv](https://docs.astral.sh/uv/) is installed, it runs + `uv tool run --from deepctl== deepctl`. uv brings its own Python. +3. Otherwise it finds Python 3.10 or newer and builds a private venv under + your cache directory (`~/.cache/deepgram-cli` or + `~/Library/Caches/deepgram-cli`). The first run takes about a minute. + +On Windows, step 1 is skipped: the launcher can't verify who controls a cache +directory there, so it never runs a venv left by an earlier run. Without uv, +every run builds a fresh venv under `%LOCALAPPDATA%\deepgram-cli` and deletes +it afterwards, which takes about a minute each time. Install uv to avoid that. +The launcher needs a real `uv.exe` or `python.exe` on `PATH`: pyenv-win's +`python.bat` shims and `uv.cmd` wrappers are skipped. + +Launcher messages go to stderr, so stdout carries only the CLI's own output +(`dg ... -o json`, `dg mcp`). + +## Prerequisites + +One of: + +- [uv](https://docs.astral.sh/uv/getting-started/installation/) (recommended): + `curl -LsSf https://astral.sh/uv/install.sh | sh`, or on Windows + `powershell -c "irm https://astral.sh/uv/install.ps1 | iex"` +- Python 3.10 or newer with `venv`. On Debian and Ubuntu that means + `sudo apt install python3-venv`. + +Node.js 20 or newer runs the launcher. + +## Already installed the CLI another way? + +If you installed the CLI with Homebrew, pip, pipx, uv or the install script, +you don't need this package. Use one install method: they all provide a +`dg` command, and two of them on the same machine fight over it. To switch to +npm, uninstall the other first (for example `brew uninstall deepgram` or +`pip uninstall deepctl`), then `npm install -g @deepgram/cli`. + +Update the npm version with `npm install -g @deepgram/cli@latest`, or run +`npx @deepgram/cli@latest`. + +## Environment variables + +| Variable | Effect | +| --- | --- | +| `DEEPCTL_NPM_RUNNER=uvx\|venv\|path` | Force one strategy. `path` runs a `deepctl` you installed yourself from your `PATH`. | +| `DEEPCTL_NPM_PYPI_SPEC` | Replace the requirement installed from PyPI (default `deepctl==`). Meant for testing. | +| `XDG_CACHE_HOME` | Cache directory base on Linux. | + +## Known limitation: nohup and ignored signals + +`nohup dg ...`, or a script that runs `dg` in the background after +`trap '' INT`, does not protect the CLI the way it protects a native program. +Node.js resets every signal its parent set to "ignore" back to the default +before the launcher's code runs, so the launcher can't tell that SIGHUP or +SIGINT was meant to be ignored, and the CLI it starts gets the defaults too. +To keep `dg` running after you close the terminal, use `setsid dg ...` (Linux), +`dg ... & disown`, or `tmux`/`screen`. + +## Documentation + +- CLI reference and source: +- Deepgram docs: diff --git a/npm/bin/dg.js b/npm/bin/dg.js new file mode 100755 index 00000000..761c717f --- /dev/null +++ b/npm/bin/dg.js @@ -0,0 +1,4 @@ +#!/usr/bin/env node +'use strict'; + +require('../lib/launcher.js').run(); diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js new file mode 100644 index 00000000..11e63256 --- /dev/null +++ b/npm/lib/launcher.js @@ -0,0 +1,1229 @@ +'use strict'; + +// Runs the Python Deepgram CLI (deepctl, from PyPI) at the same version as +// this npm package. Nothing here is downloaded at install time: the first +// `dg` either hands off to uv, reuses a cached venv, or builds one. + +const childProcess = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const tty = require('node:tty'); +const { setTimeout: delay } = require('node:timers/promises'); + +const PKG_DIR = path.resolve(__dirname, '..'); +const MIN_PYTHON = [3, 10]; +const LOCK_WAIT_MS = 5 * 60 * 1000; +const LOCK_STALE_MS = 10 * 60 * 1000; +// No first-run install takes this long; past it a same-host lock is +// abandoned even if its pid has been reused by an unrelated process. +const LOCK_MAX_MS = 60 * 60 * 1000; +const LOCK_POLL_MS = 250; +const PRUNE_AFTER_MS = 30 * 24 * 60 * 60 * 1000; +// One-run venvs (Windows) older than this were left by a run that died. +const ONE_RUN_STALE_MS = 60 * 60 * 1000; +const UV_INSTALL_WINDOWS = 'powershell -c "irm https://astral.sh/uv/install.ps1 | iex"'; +const SIGNAL_GRACE_MS = 1000; +const STDERR_EAGAIN_RETRIES = 100; +const STDERR_EAGAIN_SLEEP_MS = 10; +const PROBE_TIMEOUT_MS = 10 * 1000; +// How long an interrupted setup step gets to exit before it is killed. +const SETUP_KILL_GRACE_MS = 5 * 1000; +const RUNNERS = ['uvx', 'venv', 'path']; + +// Prints the version first so a Python without venv/ensurepip (Debian's +// python3 without python3-venv) still reports what it is. +const PROBE_CODE = 'import sys;print(tuple(sys.version_info[:2]));import venv,ensurepip'; + +// Launcher messages go straight to fd 2. Touching process.stdin, stdout or +// stderr makes Node wrap the fd in a stream, and for a pipe libuv then sets +// O_NONBLOCK on it. deepctl inherits the same open file, and a non-blocking +// stdin breaks `ffmpeg ... | dg listen` and MCP hosts that send their first +// message a moment after starting `dg mcp`. +// +// Writing is best effort: fd 2 may be a full non-blocking pipe (EAGAIN) or +// one whose reader has gone (EPIPE), and a progress line must never abort +// a first-run setup or keep deepctl from running. +function writeStderr(text, opts = {}) { + const writeSync = opts.writeSync || fs.writeSync; + const sleep = opts.sleep || sleepSync; + const buf = Buffer.from(String(text)); + let offset = 0; + let retries = 0; + while (offset < buf.length) { + try { + offset += writeSync(2, buf, offset, buf.length - offset); + } catch (err) { + if (err.code !== 'EAGAIN' || retries++ >= STDERR_EAGAIN_RETRIES) return; + sleep(STDERR_EAGAIN_SLEEP_MS); + } + } +} + +const fdStderr = { write: (text) => writeStderr(text) }; + +class LauncherError extends Error { + constructor(message, hint = '') { + super(message); + this.hint = hint; + } +} + +// --- Setup supervision ---------------------------------------------------- +// +// First-run setup (probing Python, waiting for the lock, building the venv) +// runs child processes. A signal that reaches only this process (a +// supervisor's SIGTERM, a closed terminal's SIGHUP) must not leave them +// running, or leave a lock and a half-built venv behind. So setup is async: +// run() installs its signal handlers before anything starts, a session +// tracks the one setup child running at a time, and an interrupt kills it, +// wakes any lock wait, and lets the finally blocks remove the partial venv +// and release the lock before the launcher exits 128+signo. + +class Interrupted extends Error { + constructor(signal) { + super(`setup interrupted (${signal})`); + this.signal = signal; + } +} + +function createSession() { + return { phase: 'setup', child: null, group: false, interrupted: null, abort: new AbortController(), lock: null }; +} + +function throwIfInterrupted(session) { + if (session && session.interrupted) throw new Interrupted(session.interrupted); +} + +// Setup children run in their own process group on POSIX, so the whole +// tree goes (pip's own subprocesses included) rather than just the parent. +function killSetupChild(session, sig) { + const { child, group } = session; + if (!child || child.exitCode !== null || child.signalCode !== null) return; + try { + if (group) process.kill(-child.pid, sig); + else child.kill(sig); + } catch { + try { + child.kill(sig); + } catch { + // Already gone. + } + } +} + +function interruptSetup(session, sig) { + if (session.interrupted) { + // A second signal: stop waiting for a graceful exit. + killSetupChild(session, 'SIGKILL'); + return; + } + session.interrupted = sig; + session.abort.abort(); + const child = session.child; + if (!child) return; + killSetupChild(session, sig); + const timer = setTimeout(() => { + if (session.child === child) killSetupChild(session, 'SIGKILL'); + }, SETUP_KILL_GRACE_MS); + timer.unref(); + child.once('exit', () => clearTimeout(timer)); +} + +function signalExitCode(sig) { + const num = os.constants.signals[sig]; + return num ? 128 + num : 1; +} + +// Die from `sig` ourselves, so a shell or supervisor sees the signal (a +// bash loop stops on Ctrl-C) rather than a plain exit status. `exitCode` +// (128+signo) is the fallback if the signal does not end us. +function dieBySignal(sig, exitCode) { + process.removeAllListeners(sig); + try { + process.kill(process.pid, sig); + } catch { + // Fall through to the numeric exit below. + } + setTimeout(() => process.exit(exitCode), 100); +} + +// Spawns one setup child under the session and resolves when it is gone: +// {code, signal} on exit, {error} if it could not start. +function superviseChild(command, args, options, session, spawn = childProcess.spawn) { + return new Promise((resolve) => { + let child; + try { + child = spawn(command, args, options); + } catch (error) { + resolve({ error }); + return; + } + // A real pid only: kill(-0) would signal our own process group. + const group = Boolean(options.detached) && Number.isInteger(child.pid) && child.pid > 0; + if (session) Object.assign(session, { child, group }); + // Name the step's process group in the lock, so whoever breaks the lock + // after a SIGKILLed launcher can stop the pip it left running. + if (group && session && session.lock) recordSetupGroup(session.lock, child.pid); + let settled = false; + const done = (result) => { + if (settled) return; + settled = true; + if (session && session.child === child) { + // Anything left in the group after an interrupt is a straggler. + if (group && session.interrupted) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + // Group already empty. + } + } + Object.assign(session, { child: null, group: false }); + } + resolve({ ...result, child }); + }; + child.once('error', (error) => done({ error })); + child.once('exit', (code, signal) => done({ code, signal })); + }); +} + +function packageVersion(pkgDir = PKG_DIR) { + return JSON.parse(fs.readFileSync(path.join(pkgDir, 'package.json'), 'utf8')).version; +} + +function pypiSpec(env, version) { + return env.DEEPCTL_NPM_PYPI_SPEC || `deepctl==${version}`; +} + +// Venvs are keyed by version. An overridden spec gets its own directory so +// a CI run against a different deepctl never poisons the real cache entry. +function venvKey(env, version) { + if (!env.DEEPCTL_NPM_PYPI_SPEC) return version; + const hash = crypto.createHash('sha256').update(env.DEEPCTL_NPM_PYPI_SPEC).digest('hex'); + return `${version}+${hash.slice(0, 12)}`; +} + +// --- PATH resolution ------------------------------------------------------ + +function isWindows(platform) { + return platform === 'win32'; +} + +function exeNames(name, platform) { + // No PATHEXT and no .cmd/.bat: those need a shell to run, and a shell + // re-parses the arguments we pass through. + return isWindows(platform) ? [`${name}.exe`, `${name}.com`] : [name]; +} + +function pathDirs(env, platform) { + const raw = (isWindows(platform) ? env.Path || env.PATH : env.PATH) || ''; + const p = isWindows(platform) ? path.win32 : path.posix; + // Empty and relative entries resolve against the current directory, and + // Windows searches it implicitly. A launcher run from inside an arbitrary + // project must not pick up an executable that happens to sit there. + return raw.split(p.delimiter).filter((dir) => dir && p.isAbsolute(dir)); +} + +function isExecutableFile(file, platform = process.platform) { + try { + const stat = fs.statSync(file); + if (!stat.isFile()) return false; + if (isWindows(platform)) return true; + fs.accessSync(file, fs.constants.X_OK); + return true; + } catch { + return false; + } +} + +function realpathOrSelf(file) { + try { + return fs.realpathSync(file); + } catch { + return file; + } +} + +function isInside(child, parent, platform) { + const p = isWindows(platform) ? path.win32 : path.posix; + const rel = p.relative(parent, child); + return rel === '' || (!rel.startsWith('..') && !p.isAbsolute(rel)); +} + +// Every match for `name` on PATH, in order, minus anything that resolves +// into this package (npm's own `dg` shim must never be mistaken for deepctl). +function whichAll(name, opts = {}) { + const env = opts.env || process.env; + const platform = opts.platform || process.platform; + const isExec = opts.isExecutable || ((f) => isExecutableFile(f, platform)); + const realpath = opts.realpath || realpathOrSelf; + const pkgDir = realpath(opts.pkgDir || PKG_DIR); + const join = isWindows(platform) ? path.win32.join : path.posix.join; + const found = []; + for (const dir of pathDirs(env, platform)) { + for (const exe of exeNames(name, platform)) { + const candidate = join(dir, exe); + if (!isExec(candidate)) continue; + if (isInside(realpath(candidate), pkgDir, platform)) continue; + if (!found.includes(candidate)) found.push(candidate); + } + } + return found; +} + +function which(name, opts) { + return whichAll(name, opts)[0] || null; +} + +function isWindowsAppsStub(file) { + return /[\\/]WindowsApps[\\/]/i.test(file); +} + +// --- uv ------------------------------------------------------------------- + +function findUv(opts = {}) { + const platform = opts.platform || process.platform; + const home = opts.home || os.homedir(); + const isExec = opts.isExecutable || ((f) => isExecutableFile(f, platform)); + const onPath = which('uv', opts); + if (onPath) return onPath; + // The uv installer drops it here but only edits shell profiles, so a + // fresh install is often not on PATH in the shell npm was started from. + const join = isWindows(platform) ? path.win32.join : path.posix.join; + const exe = isWindows(platform) ? 'uv.exe' : 'uv'; + for (const dir of [join(home, '.local', 'bin'), join(home, '.cargo', 'bin')]) { + const candidate = join(dir, exe); + if (isExec(candidate)) return candidate; + } + return null; +} + +// --- Python --------------------------------------------------------------- + +function pythonCandidates(opts = {}) { + const platform = opts.platform || process.platform; + const candidates = []; + if (isWindows(platform)) { + // WindowsApps holds both the Store's "python" stub, which only prints + // an install hint, and real Store-installed Pythons. Try it last and let + // the version probe tell them apart. + const pys = whichAll('py', opts); + const pythons = whichAll('python', opts); + const ordered = [ + ...pys.filter((f) => !isWindowsAppsStub(f)).map((f) => ({ f, py: true })), + ...pythons.filter((f) => !isWindowsAppsStub(f)).map((f) => ({ f, py: false })), + ...pys.filter(isWindowsAppsStub).map((f) => ({ f, py: true })), + ...pythons.filter(isWindowsAppsStub).map((f) => ({ f, py: false })), + ]; + for (const { f, py } of ordered) { + if (!py) { + candidates.push({ command: f, args: [] }); + continue; + } + for (const minor of [13, 12, 11, 10]) candidates.push({ command: f, args: [`-3.${minor}`] }); + // Newer than 3.13, or a launcher configured with only a default. + candidates.push({ command: f, args: ['-3'] }); + } + return candidates; + } + for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) { + for (const exe of whichAll(name, opts)) { + // Running Apple's stub pops the "install command line developer + // tools" dialog instead of failing quietly. + if (platform === 'darwin' && exe === '/usr/bin/python3') continue; + if (!candidates.some((c) => c.command === exe)) candidates.push({ command: exe, args: [] }); + } + } + return candidates; +} + +function parseProbeOutput(stdout) { + const match = /\((\d+),\s*(\d+)\)/.exec(String(stdout || '')); + return match ? [Number(match[1]), Number(match[2])] : null; +} + +function versionAtLeast(version, minimum) { + return version[0] > minimum[0] || (version[0] === minimum[0] && version[1] >= minimum[1]); +} + +// Runs the probe and collects its stdout: {status, stdout}. +async function execProbe(command, args, session) { + let stdout = ''; + let closed = Promise.resolve(); + const spawn = (cmd, argv, options) => { + const child = childProcess.spawn(cmd, argv, options); + const timer = setTimeout(() => child.kill('SIGKILL'), PROBE_TIMEOUT_MS); + // 'exit' can come before the last of stdout; 'close' comes after it. + closed = new Promise((resolve) => { + child.once('close', resolve); + child.once('error', resolve); + }).finally(() => clearTimeout(timer)); + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (d) => (stdout += d)); + return child; + }; + const result = await superviseChild(command, args, { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true }, session, spawn); + await closed; + return { status: result.error ? null : result.code, stdout }; +} + +async function probePython(candidate, exec = execProbe) { + const result = await exec(candidate.command, [...candidate.args, '-c', PROBE_CODE]); + const version = parseProbeOutput(result.stdout); + if (!version) return { version: null, ok: false }; + return { version, ok: result.status === 0 && versionAtLeast(version, MIN_PYTHON), hasVenv: result.status === 0 }; +} + +// Returns the first usable Python plus what was seen on the way, so the +// failure message can say "found 3.9" or "found 3.12 without venv". +async function findPython(opts = {}) { + const session = opts.session; + const probe = opts.probe || ((c) => probePython(c, (cmd, args) => execProbe(cmd, args, session))); + const seen = []; + for (const candidate of pythonCandidates(opts)) { + throwIfInterrupted(session); + const info = await probe(candidate); + throwIfInterrupted(session); + if (!info.version) continue; + seen.push({ ...candidate, ...info }); + if (info.ok) return { python: candidate, seen }; + } + return { python: null, seen }; +} + +// --- Cache directory ------------------------------------------------------ + +function cacheRootSpecs(opts = {}) { + const env = opts.env || process.env; + const platform = opts.platform || process.platform; + const home = opts.home || os.homedir(); + const tmpdir = opts.tmpdir || os.tmpdir(); + const uid = opts.uid !== undefined ? opts.uid : process.getuid ? process.getuid() : 'user'; + const p = isWindows(platform) ? path.win32 : path.posix; + let base; + if (isWindows(platform)) { + // Windows only ever builds one-run venvs (see cacheReuse), so there is + // no shared %TEMP% fallback to trust: a predictable name in a directory + // others may reach is exactly what cannot be validated there. + base = env.LOCALAPPDATA && p.isAbsolute(env.LOCALAPPDATA) ? env.LOCALAPPDATA : p.join(home, 'AppData', 'Local'); + return [{ dir: p.join(base, 'deepgram-cli', 'venvs'), shared: false }]; + } + if (platform === 'darwin') base = p.join(home, 'Library', 'Caches'); + else base = env.XDG_CACHE_HOME && p.isAbsolute(env.XDG_CACHE_HOME) ? env.XDG_CACHE_HOME : p.join(home, '.cache'); + // The fallback covers containers run as an arbitrary UID with a HOME that + // does not exist or is read-only. It sits in a directory every user can + // write to, so it is held to a stricter ownership check (see rootTrusted). + return [ + { dir: p.join(base, 'deepgram-cli', 'venvs'), shared: false }, + { dir: p.join(tmpdir, `deepgram-cli-${uid}`, 'venvs'), shared: true }, + ]; +} + +function cacheRoots(opts = {}) { + return cacheRootSpecs(opts).map((spec) => spec.dir); +} + +// Whether a venv left by an earlier run may be executed again. Reuse is only +// safe when the cache root can be shown to be ours (rootTrusted). On Windows +// that would mean checking owners, ACLs and reparse points on every +// ancestor, which Node cannot do, so Windows never reuses a venv: it runs +// through uv, or builds a private venv for the one run and deletes it. +function cacheReuse(opts = {}) { + if (opts.cacheReuse !== undefined) return opts.cacheReuse; + return !isWindows(opts.platform || process.platform); +} + +function currentUid(opts) { + if (opts.uid !== undefined && typeof opts.uid === 'number') return opts.uid; + return process.getuid ? process.getuid() : null; +} + +const STICKY = 0o1000; + +// Whatever is in a cache root gets executed, so the root and its parent +// must be ours: real directories (not symlinks) owned by this uid that no +// one else can write to. In a shared root such as /tmp, another user could +// otherwise pre-create the directory and plant a "cached" deepctl, so it +// must also be private (0700). +// +// Every ancestor above them matters too: whoever can rename entries in an +// ancestor can swap our cache for theirs between our check and our exec. +// So each ancestor must belong to root or to us, and if others can write to +// it, it must be sticky (like /tmp), which stops them renaming our entries. +// Windows has no uid, and its ACLs and reparse points are out of reach from +// Node, so no Windows root is trusted (see cacheReuse). +function rootTrusted(spec, opts = {}) { + const platform = opts.platform || process.platform; + if (isWindows(platform)) return false; + const uid = currentUid(opts); + const lstat = opts.lstat || fs.lstatSync; + const forbidden = spec.shared ? 0o077 : 0o022; + const statDir = (dir) => { + try { + const st = lstat(dir); + return st.isDirectory() ? st : null; + } catch { + return null; + } + }; + const top = path.posix.dirname(spec.dir); + for (const dir of [top, spec.dir]) { + const st = statDir(dir); + if (!st || st.uid !== uid || (st.mode & forbidden) !== 0) return false; + } + // Both the path as written (whose symlinks, like macOS's /var, must not + // be replaceable) and the real path it resolves to. + const realpath = opts.realpath || fs.realpathSync; + const above = path.posix.dirname(top); + let real; + try { + real = realpath(above); + } catch { + return false; + } + const gid = opts.gid !== undefined ? opts.gid : process.getgid ? process.getgid() : null; + return ancestorsSafe(above, uid, gid, lstat, true) && ancestorsSafe(real, uid, gid, lstat, false); +} + +// Group-writable is trusted only for a directory we own whose group id +// equals our uid: the user-private-group convention (Ubuntu, Fedora; umask +// 002), where ~/.cache is 0775 with a group named after us. A shared +// primary group (SUSE and Debian USERGROUPS=no use `users`, macOS uses +// `staff`) has other members who could swap the cache during the first +// build, so anything else group-writable fails closed. +function ancestorsSafe(start, uid, gid, lstat, allowLinks) { + for (let dir = start; ; dir = path.posix.dirname(dir)) { + let st; + try { + st = lstat(dir); + } catch { + return false; + } + const link = st.isSymbolicLink ? st.isSymbolicLink() : false; + if (link ? !allowLinks : !st.isDirectory()) return false; + if (st.uid !== 0 && st.uid !== uid) return false; + const privateGroup = st.uid === uid && st.gid === gid && gid === uid; + const writableByOthers = (st.mode & 0o002) !== 0 || ((st.mode & 0o020) !== 0 && !privateGroup); + if (!link && writableByOthers && (st.mode & STICKY) === 0) return false; + if (dir === '/' || dir === '.') return true; + } +} + +function mkdirPrivate(dir) { + try { + fs.mkdirSync(dir, { mode: 0o700 }); + } catch (err) { + if (err.code !== 'EEXIST') throw err; + } +} + +// Creates the root if needed and returns whether it is safe and writable. +function prepareRoot(spec, opts = {}) { + const platform = opts.platform || process.platform; + const p = isWindows(platform) ? path.win32 : path.posix; + const top = p.dirname(spec.dir); + try { + // 0700 so that, under umask 002, we do not create a group-writable + // ~/.cache ourselves. + fs.mkdirSync(p.dirname(top), { recursive: true, mode: 0o700 }); + mkdirPrivate(top); + mkdirPrivate(spec.dir); + if (!rootTrusted(spec, opts)) return false; + fs.accessSync(spec.dir, fs.constants.W_OK); + return true; + } catch { + return false; + } +} + +function venvLayout(dir, platform = process.platform) { + const p = isWindows(platform) ? path.win32 : path.posix; + const bin = isWindows(platform) ? p.join(dir, 'Scripts') : p.join(dir, 'bin'); + const ext = isWindows(platform) ? '.exe' : ''; + return { + dir, + marker: p.join(dir, '.complete'), + python: p.join(bin, `python${ext}`), + deepctl: p.join(bin, `deepctl${ext}`), + }; +} + +function findCachedVenv(key, opts = {}) { + if (!cacheReuse(opts)) return null; + const exists = opts.exists || fs.existsSync; + const trusted = opts.trustRoot || ((spec) => rootTrusted(spec, opts)); + const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; + for (const spec of cacheRootSpecs(opts)) { + if (!trusted(spec)) continue; + const layout = venvLayout(p.join(spec.dir, key), opts.platform); + if (venvUsable(layout, exists)) return layout; + } + return null; +} + +// A venv's python is a link to the interpreter that built it. When that +// interpreter is uninstalled (brew, pyenv, a distro upgrade), every console +// script fails with a confusing ENOENT, so a venv without a working python +// is treated as missing and rebuilt. existsSync follows the link. +function venvUsable(layout, exists = fs.existsSync) { + return exists(layout.marker) && exists(layout.deepctl) && exists(layout.python); +} + +// Marks a venv as recently used, which keeps pruneSiblings away from it. +function touchMarker(marker) { + try { + const now = new Date(); + fs.utimesSync(marker, now, now); + } catch { + // Best effort: a read-only cache only loses the recency hint. + } +} + +// --- Lock ----------------------------------------------------------------- + +// Only for retrying a full stderr pipe, never for waiting on setup. +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function pidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return err.code === 'EPERM'; + } +} + +// owner.json is only ever replaced whole (write a temp file, rename it +// over), so a reader sees the old or the new owner, never a torn one. +// Every write goes through here. +function writeOwner(lockDir, owner, writeFile = fs.writeFileSync) { + const file = path.join(lockDir, 'owner.json'); + const tmp = `${file}.${process.pid}.tmp`; + try { + writeFile(tmp, JSON.stringify(owner)); + fs.renameSync(tmp, file); + } catch (err) { + try { + fs.rmSync(tmp, { force: true }); + } catch { + // Removed with the lock directory anyway. + } + throw err; + } +} + +// {owner, raw} when owner.json holds an owner, else {owner: null}: missing +// (the owner is between mkdir and its first write) or unreadable (written +// non-atomically by an older launcher, or not JSON at all). +function readOwner(lockDir) { + let raw; + try { + raw = fs.readFileSync(path.join(lockDir, 'owner.json'), 'utf8'); + } catch { + return { owner: null, raw: null }; + } + try { + const owner = JSON.parse(raw); + if (owner && typeof owner.pid === 'number') return { owner, raw }; + } catch { + // Fall through. + } + return { owner: null, raw }; +} + +// Gives a missing or unreadable owner.json a moment to appear before the +// caller falls back to judging the lock by its directory's age. +async function readOwnerSettled(lockDir, tries = 5, gapMs = 20) { + for (let i = 1; ; i++) { + const read = readOwner(lockDir); + if (read.owner || i >= tries) return read; + await delay(gapMs); + } +} + +function lockIsStale(lockDir, now = Date.now(), alive = pidAlive, read = readOwner(lockDir)) { + // No owner (see readOwner): fall back to the directory's age. + const owner = read.owner; + if (owner) { + if (owner.host === os.hostname()) { + // Our own pid in someone else's lock means the pid was reused: a + // container that crashed mid-setup restarts as pid 1 with the same + // hostname and cache volume. A live pid is otherwise trusted for a + // slow install, up to an hour, after which it must be a reused pid. + if (owner.pid === process.pid) return true; + if (!alive(owner.pid)) return true; + return now - owner.time > LOCK_MAX_MS; + } + // Across hosts (a shared home directory) the pid means nothing. + return now - owner.time > LOCK_STALE_MS; + } + try { + return now - fs.statSync(lockDir).mtimeMs > LOCK_STALE_MS; + } catch { + return false; + } +} + +// mkdir is atomic on every platform and filesystem npm runs on, unlike +// O_EXCL files on some network mounts. The wait between polls is a timer, +// not a blocking sleep, so a signal can end it (opts.session). +async function acquireLock(lockDir, opts = {}) { + const session = opts.session; + const waitMs = opts.waitMs !== undefined ? opts.waitMs : LOCK_WAIT_MS; + const pollMs = opts.pollMs || LOCK_POLL_MS; + const onWait = opts.onWait || (() => {}); + const deadline = Date.now() + waitMs; + let announced = false; + const writeFile = opts.writeFile || fs.writeFileSync; + for (;;) { + throwIfInterrupted(session); + let created = false; + try { + fs.mkdirSync(lockDir); + created = true; + const owner = { pid: process.pid, host: os.hostname(), time: Date.now() }; + writeOwner(lockDir, owner, writeFile); + const release = () => releaseLock(lockDir, owner); + release.lock = { dir: lockDir, owner }; + return release; + } catch (err) { + if (created) fs.rmSync(lockDir, { recursive: true, force: true }); + if (err.code !== 'EEXIST' || created) { + throw new LauncherError(`Could not create the setup lock ${lockDir}: ${err.message}`); + } + } + if (lockIsStale(lockDir, Date.now(), pidAlive, await readOwnerSettled(lockDir))) { + breakStaleLock(lockDir); + continue; + } + if (Date.now() >= deadline) { + throw new LauncherError(`Timed out waiting for another dg setup to finish (lock: ${lockDir}).`); + } + if (!announced) { + onWait(); + announced = true; + } + try { + await delay(pollMs, undefined, session ? { signal: session.abort.signal } : undefined); + } catch (err) { + if (err.name !== 'AbortError') throw err; + } + } +} + +function readOwnerRaw(lockDir) { + return readOwner(lockDir).raw; +} + +// Two waiters can both judge the same lock stale. If each simply deleted +// it, the slower one could delete the lock the faster one has just taken. +// Instead, move the lock aside under a unique name (rename is atomic, so +// only one waiter gets the stale lock) and delete it only if what moved is +// still the owner that was judged stale. If a fresh lock was moved by +// mistake, try to put it back; the caller then simply retries. +// +// A residual interleaving remains with three or more waiters (one restores +// a lock another has already replaced). Directory locks cannot close it +// completely, and it needs a stale lock plus three concurrent first runs; +// the worst case is two builds of the same venv racing, which is accepted. +function breakStaleLock(lockDir) { + const judged = readOwnerRaw(lockDir); + const grave = `${lockDir}.stale.${process.pid}.${crypto.randomBytes(4).toString('hex')}`; + try { + fs.renameSync(lockDir, grave); + } catch { + return; // Someone else moved or removed it first. + } + if (readOwnerRaw(grave) === judged) { + killOrphanedSetup(judged, lockDir.replace(/\.lock$/, '')); + fs.rmSync(grave, { recursive: true, force: true }); + return; + } + try { + fs.renameSync(grave, lockDir); + } catch { + // A new lock exists already; leave the moved one for pruning. + } +} + +function recordSetupGroup(lock, pgid) { + try { + lock.owner.pgid = pgid; + writeOwner(lock.dir, lock.owner); + } catch { + // Best effort: without it a stale-lock breaker just cannot stop the group. + } +} + +// A launcher killed with SIGKILL cannot stop its setup step, whose pip keeps +// writing into the venv directory the next run is about to rebuild. The +// stale lock names that step's process group; kill it, but only on Linux +// (where /proc can show it) and only if some process in the group still +// mentions the venv directory on its command line, so a pgid reused by an +// unrelated process is left alone. +// The directory itself or a path inside it, never a sibling that merely +// shares a prefix (`venvs/0.4.1` must not match `venvs/0.4.10`). +function mentionsDir(arg, dir) { + return arg === dir || arg.startsWith(dir + path.sep); +} + +function killOrphanedSetup(ownerRaw, venvDir, procDir = '/proc') { + let owner; + try { + owner = JSON.parse(ownerRaw); + } catch { + return false; + } + const pgid = owner && owner.pgid; + if (!Number.isInteger(pgid) || pgid <= 1 || owner.host !== os.hostname()) return false; + let pids; + try { + pids = fs.readdirSync(procDir).filter((e) => /^\d+$/.test(e)); + } catch { + return false; // No /proc (macOS): nothing safe to check against. + } + const ours = pids.some((pid) => { + try { + const stat = fs.readFileSync(path.join(procDir, pid, 'stat'), 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + if (Number(fields[2]) !== pgid) return false; + return fs.readFileSync(path.join(procDir, pid, 'cmdline'), 'utf8').split('\0').some((arg) => mentionsDir(arg, venvDir)); + } catch { + return false; + } + }); + if (!ours) return false; + try { + process.kill(-pgid, 'SIGKILL'); + } catch { + // ESRCH: it exited on its own. + } + return true; +} + +// Only removes the lock if it is still ours: if it was judged stale and +// taken over while we ran, the new owner's lock must survive. +function releaseLock(lockDir, owner) { + // Gone already, or not readable: either way it is not ours to remove. + const current = readOwner(lockDir).owner; + if (current && current.pid === owner.pid && current.time === owner.time && current.host === owner.host) { + try { + fs.rmSync(lockDir, { recursive: true, force: true }); + } catch { + // Best effort: a lock left behind is judged stale by the next run. + } + } +} + +// --- Venv bootstrap ------------------------------------------------------- + +const CRASH_SIGNALS = new Set(['SIGSEGV', 'SIGBUS', 'SIGILL', 'SIGFPE', 'SIGABRT']); + +async function runStep(command, args, what, hint, opts = {}) { + const session = opts.session; + throwIfInterrupted(session); + // stdout goes to stderr: `dg -o json` and `dg mcp` own stdout. A process + // group of its own (POSIX) lets an interrupt take down pip's children too. + const options = { stdio: ['ignore', 2, 2], windowsHide: true, detached: !isWindows(process.platform) }; + const result = await superviseChild(command, args, options, session, opts.spawn); + throwIfInterrupted(session); + if (result.error) throw new LauncherError(`${what} failed: ${result.error.message}.`, hint); + if (result.code !== 0) { + const how = result.signal ? `killed by ${result.signal}` : `exit ${result.code}`; + throw new LauncherError(`${what} failed (${how}).`, hint); + } +} + +const VENV_HINT = 'On Debian or Ubuntu, install python3-venv (sudo apt install python3-venv), or install uv instead.'; +const NETWORK_HINT = 'Check your network connection and that PyPI is reachable, then run dg again.'; + +// Removes other versions' venvs, but only ones nobody has used for a long +// time: a dg pinned to an older npm version (npx @deepgram/cli@0.4.0, a +// project-local install) may be running from one right now. Every run from +// a cached venv refreshes its .complete mtime. +function pruneSiblings(root, keep, now = Date.now()) { + let entries; + try { + entries = fs.readdirSync(root); + } catch { + return; + } + for (const entry of entries) { + if (entry === keep || entry.endsWith('.lock')) continue; + try { + const lock = path.join(root, `${entry}.lock`); + if (fs.existsSync(lock) && !lockIsStale(lock, now)) continue; + const dir = path.join(root, entry); + let idle; + try { + idle = now - fs.statSync(path.join(dir, '.complete')).mtimeMs > PRUNE_AFTER_MS; + } catch { + // No marker: a build that died. Leave it until no live build could own it. + idle = now - fs.statSync(dir).mtimeMs > LOCK_STALE_MS; + } + if (idle) fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // Best effort: an old version left behind only costs disk space. + } + } +} + +// Builds a venv for this run only, in a directory created fresh with an +// unpredictable name, so nothing another process prepared can be picked up. +// The caller deletes it once deepctl exits. +async function buildPrivateVenv({ python, key, spec, version, opts = {} }) { + const stderr = opts.stderr || fdStderr; + const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; + const base = cacheRootSpecs(opts)[0].dir; + let dir; + try { + fs.mkdirSync(base, { recursive: true, mode: 0o700 }); + pruneOneRunVenvs(base, p); + dir = fs.mkdtempSync(p.join(base, `${key}-`)); + } catch (err) { + throw new LauncherError(`No writable directory for the Deepgram CLI: ${err.message}`, 'Install uv instead: it needs no setup on each run.'); + } + const layout = venvLayout(dir, opts.platform); + stderr.write(`Setting up Deepgram CLI ${version} for this run (~1 min each time; install uv to skip it: ${UV_INSTALL_WINDOWS})...\n`); + try { + await installInto(python, layout, spec, opts); + } catch (err) { + fs.rmSync(dir, { recursive: true, force: true }); + throw err; + } + return { ...layout, ephemeral: true }; +} + +// A one-run venv outlives its run when cleanup never happens: Node +// crashed, the process was killed (Task Manager, taskkill /F), or a closing +// console's few seconds of grace ran out before deepctl exited. Remove those, but only once they are old enough that no setup +// could still be building them, and only if they can be moved aside first: +// Windows refuses to rename a directory while a running deepctl has files +// open in it, so a long session's venv survives. +function pruneOneRunVenvs(base, p = path, now = Date.now()) { + let entries; + try { + entries = fs.readdirSync(base); + } catch { + return; + } + for (const entry of entries) { + // mkdtemp names (-XXXXXX) and ones a previous prune moved aside. + if (!/-[A-Za-z0-9]{6}(\.del)?$/.test(entry)) continue; + const dir = p.join(base, entry); + try { + if (now - fs.statSync(dir).mtimeMs <= ONE_RUN_STALE_MS) continue; + const grave = entry.endsWith('.del') ? dir : `${dir}.del`; + if (grave !== dir) fs.renameSync(dir, grave); + fs.rmSync(grave, { recursive: true, force: true }); + } catch { + // In use, or already gone: leave it for a later run. + } + } +} + +async function installInto(python, layout, spec, opts) { + const pip = [layout.python, '-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q']; + await runStep(python.command, [...python.args, '-m', 'venv', layout.dir], 'Creating the Python venv', VENV_HINT, opts); + await runStep(pip[0], [...pip.slice(1), '-U', 'pip'], 'Upgrading pip', NETWORK_HINT, opts); + await runStep(pip[0], [...pip.slice(1), spec], `Installing ${spec}`, NETWORK_HINT, opts); +} + +// The lock is released in `finally`, not from process.on('exit'): that hook +// never runs when the process dies from a signal. run() turns signals into +// an Interrupted error instead, so this unwinds normally. +async function buildVenv({ python, key, spec, version, opts = {} }) { + if (!cacheReuse(opts)) return buildPrivateVenv({ python, key, spec, version, opts }); + const stderr = opts.stderr || fdStderr; + const prepare = opts.prepareRoot || ((spec) => prepareRoot(spec, opts)); + const rootSpec = cacheRootSpecs(opts).find(prepare); + const root = rootSpec && rootSpec.dir; + if (!root) throw new LauncherError('No writable cache directory for the Deepgram CLI.', 'Set XDG_CACHE_HOME to a writable directory, or install uv.'); + // Built in place: venv console scripts carry absolute shebangs, so a venv + // built elsewhere and renamed into place would not run. + const layout = venvLayout(path.join(root, key), opts.platform); + const release = await acquireLock(`${layout.dir}.lock`, { + ...opts.lock, + session: opts.session, + onWait: () => stderr.write('Waiting for another dg setup to finish...\n'), + }); + if (opts.session) opts.session.lock = release.lock; + try { + if (venvUsable(layout)) return layout; + // Half-built, or broken by a removed base interpreter: start over. + fs.rmSync(layout.dir, { recursive: true, force: true }); + stderr.write(`Setting up Deepgram CLI ${version} (one-time, ~1 min)...\n`); + try { + await installInto(python, layout, spec, opts); + fs.writeFileSync(layout.marker, `${spec}\n`); + } catch (err) { + fs.rmSync(layout.dir, { recursive: true, force: true }); + throw err; + } + pruneSiblings(root, key); + return layout; + } finally { + if (opts.session) opts.session.lock = null; + release(); + } +} + +// --- Child environment and exit mapping ------------------------------------ + +// npx and `npm exec` unpack packages under npm's cache in a `_npx` +// directory. Where this package lives is what counts, not how the parent +// was started: a global dg run from inside an npx process is still global. +function launcherKind(pkgDir = PKG_DIR) { + return pkgDir.split(/[\\/]/).includes('_npx') ? 'npx' : 'npm'; +} + +function childEnv(env, strategy, pkgDir = PKG_DIR) { + const out = { ...env }; + // A deepctl the user installed themselves knows its own install method; + // telling it "npm" would make `dg update` give the wrong advice. + if (strategy === 'path') delete out.DEEPCTL_LAUNCHER; + else out.DEEPCTL_LAUNCHER = launcherKind(pkgDir); + return out; +} + +function exitAction(code, signal, platform = process.platform) { + if (code !== null && code !== undefined) return { exit: code }; + if (signal && !isWindows(platform)) { + const num = os.constants.signals[signal]; + return { signal, exit: num ? 128 + num : 1 }; + } + return { exit: 1 }; +} + +// --- Plan ----------------------------------------------------------------- + +function installHelp(seen) { + const lines = ['', 'The Deepgram CLI needs uv or Python 3.10+ (with venv) to run. Fix one of:', '']; + lines.push(' 1. Install uv (recommended), then run dg again:'); + lines.push(' curl -LsSf https://astral.sh/uv/install.sh | sh'); + lines.push(` ${UV_INSTALL_WINDOWS} (Windows)`); + lines.push(' On Windows, the launcher only runs real uv.exe / python.exe files: pyenv-win\'s'); + lines.push(' python.bat shims and uv.cmd are skipped, since a .bat or .cmd needs a shell'); + lines.push(' that would re-parse your arguments.'); + const noVenv = seen.find((s) => s.version && versionAtLeast(s.version, MIN_PYTHON) && !s.hasVenv); + if (noVenv) { + lines.push(` 2. Python ${noVenv.version.join('.')} was found without venv/ensurepip. Install it:`); + lines.push(' sudo apt install python3-venv (Debian/Ubuntu)'); + } else { + const old = seen.find((s) => s.version && !versionAtLeast(s.version, MIN_PYTHON)); + const found = old ? ` (found ${old.version.join('.')})` : ''; + lines.push(` 2. Install Python 3.10 or newer${found}.`); + } + lines.push(' 3. Or install the CLI directly:'); + lines.push(' curl -fsSL https://deepgram.com/install.sh | sh'); + lines.push(' brew install deepgram/tap/deepgram'); + return `${lines.join('\n')}\n`; +} + +// Decides how to run deepctl. Resolves to {command, args, strategy}, plus +// {cleanup} for a one-run venv; may build a venv as a side effect. +async function resolveRunner(argv, opts = {}) { + const env = opts.env || process.env; + const version = opts.version || packageVersion(); + const spec = pypiSpec(env, version); + const key = venvKey(env, version); + const forced = env.DEEPCTL_NPM_RUNNER || ''; + if (forced && !RUNNERS.includes(forced)) { + throw new LauncherError(`DEEPCTL_NPM_RUNNER must be one of ${RUNNERS.join(', ')} (got "${forced}").`); + } + + if (forced === 'path') { + const deepctl = which('deepctl', opts); + if (!deepctl) throw new LauncherError('DEEPCTL_NPM_RUNNER=path, but no deepctl was found on PATH.'); + return { command: deepctl, args: argv, strategy: 'path' }; + } + + if (!forced || forced === 'venv') { + const cached = findCachedVenv(key, opts); + if (cached) { + touchMarker(cached.marker); + return { command: cached.deepctl, args: argv, strategy: 'venv' }; + } + } + + if (!forced || forced === 'uvx') { + const uv = findUv(opts); + if (uv) return { command: uv, args: ['tool', 'run', '--from', spec, 'deepctl', ...argv], strategy: 'uvx' }; + if (forced) throw new LauncherError('DEEPCTL_NPM_RUNNER=uvx, but uv was not found.'); + } + + const { python, seen } = await findPython(opts); + if (!python) throw new LauncherError(installHelp(seen)); + const layout = await buildVenv({ python, key, spec, version, opts }); + const runner = { command: layout.deepctl, args: argv, strategy: 'venv' }; + if (layout.ephemeral) runner.cleanup = layout.dir; + return runner; +} + +async function run(argv = process.argv.slice(2), opts = {}) { + const env = opts.env || process.env; + const stderr = opts.stderr || fdStderr; + const session = createSession(); + + // Handlers go in before any setup starts. During setup, any of these + // signals interrupts it (setup children have their own process group, so + // only we can tell them). Once deepctl runs: + // + // TERM and HUP come from whoever manages *this* process: pass them on. + // Ctrl-C and Ctrl-\ from a terminal reach the child directly (same + // process group), so forwarding them at once would deliver them twice. + // But an IDE, supervisor or `timeout -s INT` signals only our pid; if the + // child is still running after a grace period, it never got the signal. + // + // With a terminal attached, assume the signal came from it and reached the + // child too: a second, forwarded interrupt would cut short a child that + // is still cleaning up after the first. On Windows a console Ctrl-C + // reaches every attached process, and child.kill() would terminate the + // child outright rather than interrupt it. + // isatty(), not process.stdin.isTTY: see fdStderr for why. + const terminal = [0, 1, 2].some((fd) => tty.isatty(fd)); + const timers = new Set(); + const childRunning = (child) => child.exitCode === null && child.signalCode === null; + const handle = (sig, forwardNow) => () => { + if (session.phase === 'setup') { + interruptSetup(session, sig); + return; + } + const child = session.child; + if (!child) return; + if (forwardNow) { + child.kill(sig); + return; + } + if (isWindows(process.platform) || terminal) return; + const timer = setTimeout(() => { + timers.delete(timer); + if (childRunning(child)) child.kill(sig); + }, SIGNAL_GRACE_MS); + timers.add(timer); + }; + // Known limitation: a signal the parent set to "ignore" (nohup, or + // `trap '' INT` before a background job) cannot be passed on. Node resets + // inherited SIG_IGN dispositions to the default before any JS runs, so + // neither /proc/self/status nor ps can see them here, and deepctl starts + // with the defaults. See "Known limitation" in README.md. + // + // On Windows, Node emits SIGHUP when the console window closes (Windows + // ends the process a few seconds later) and SIGBREAK on Ctrl-Break. + // Handling both is what lets a one-run venv be removed on the way out. + const handlers = { SIGINT: handle('SIGINT', false), SIGTERM: handle('SIGTERM', true), SIGHUP: handle('SIGHUP', true) }; + if (isWindows(process.platform)) handlers.SIGBREAK = handle('SIGBREAK', false); + else handlers.SIGQUIT = handle('SIGQUIT', false); + for (const [sig, fn] of Object.entries(handlers)) process.on(sig, fn); + const removeHandlers = () => { + for (const [sig, fn] of Object.entries(handlers)) process.removeListener(sig, fn); + }; + + // A one-run venv (Windows without uv) goes once deepctl is done with it. + let runner; + const cleanup = () => { + if (!runner || !runner.cleanup) return; + try { + fs.rmSync(runner.cleanup, { recursive: true, force: true, maxRetries: 3 }); + } catch { + // Best effort: a leftover one-run venv only costs disk space. + } + }; + + try { + runner = await resolveRunner(argv, { ...opts, session }); + throwIfInterrupted(session); + } catch (err) { + removeHandlers(); + cleanup(); + if (session.interrupted) { + // The setup child is gone, its partial venv removed and the lock + // released (buildVenv's finally); now die from the same signal. + const sig = session.interrupted; + stderr.write(`dg: setup interrupted (${sig}).\n`); + if (isWindows(process.platform)) process.exit(signalExitCode(sig)); + dieBySignal(sig, signalExitCode(sig)); + return; + } + // Filesystem surprises (a cache directory changing underneath us) get + // the same one-line treatment as expected failures, not a stack trace. + const hint = err instanceof LauncherError && err.hint ? `\n${err.hint}` : ''; + stderr.write(`dg: ${String(err.message || err).trimStart()}${hint}\n`); + process.exit(1); + } + + // No await between here and spawn: no signal can slip in unhandled. + session.phase = 'run'; + const child = childProcess.spawn(runner.command, runner.args, { + stdio: 'inherit', + env: childEnv(env, runner.strategy), + }); + session.child = child; + + child.on('error', (err) => { + stderr.write(`dg: could not start ${runner.command}: ${err.message}\n`); + cleanup(); + process.exit(1); + }); + + child.on('exit', (code, signal) => { + cleanup(); + for (const timer of timers) clearTimeout(timer); + removeHandlers(); + const action = exitAction(code, signal); + // A crash otherwise surfaces only as a bare exit status (139 and so on). + if (CRASH_SIGNALS.has(signal)) stderr.write(`dg: ${runner.command} crashed (${signal}).\n`); + if (action.signal) { + // Die the same way the child did. + dieBySignal(action.signal, action.exit); + return; + } + process.exit(action.exit); + }); +} + +module.exports = { + Interrupted, + LauncherError, + acquireLock, + cacheReuse, + cacheRootSpecs, + pruneOneRunVenvs, + cacheRoots, + breakStaleLock, + childEnv, + launcherKind, + exitAction, + exeNames, + findCachedVenv, + killOrphanedSetup, + readOwner, + writeOwner, + mentionsDir, + findPython, + findUv, + installHelp, + lockIsStale, + packageVersion, + parseProbeOutput, + probePython, + pathDirs, + writeStderr, + prepareRoot, + pypiSpec, + pruneSiblings, + pythonCandidates, + releaseLock, + resolveRunner, + rootTrusted, + run, + venvKey, + venvLayout, + venvUsable, + versionAtLeast, + which, + whichAll, +}; diff --git a/npm/package.json b/npm/package.json new file mode 100644 index 00000000..165efa9e --- /dev/null +++ b/npm/package.json @@ -0,0 +1,42 @@ +{ + "name": "@deepgram/cli", + "version": "0.4.0", + "description": "The official Deepgram CLI: speech-to-text, text-to-speech, voice agents and audio intelligence from your terminal", + "type": "commonjs", + "bin": { + "dg": "bin/dg.js", + "deepgram": "bin/dg.js" + }, + "files": [ + "bin/", + "lib/" + ], + "engines": { + "node": ">=20" + }, + "scripts": { + "test": "node --test test/launcher.test.js" + }, + "keywords": [ + "deepgram", + "cli", + "speech-to-text", + "text-to-speech", + "transcription", + "voice-agent" + ], + "homepage": "https://github.com/deepgram/cli#readme", + "bugs": { + "url": "https://github.com/deepgram/cli/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/deepgram/cli.git", + "directory": "npm" + }, + "license": "MIT", + "author": "Deepgram ", + "publishConfig": { + "access": "public" + } +} diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js new file mode 100644 index 00000000..9be2ab03 --- /dev/null +++ b/npm/test/launcher.test.js @@ -0,0 +1,1224 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { test, describe } = require('node:test'); + +const launcher = require('../lib/launcher.js'); + +const PKG_DIR = path.resolve(__dirname, '..'); +const BIN = path.join(PKG_DIR, 'bin', 'dg.js'); + +function tmpdir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'dg-launcher-test-')); +} + +function fakeExe(dir, name, body = '#!/bin/sh\nexit 0\n') { + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, name); + fs.writeFileSync(file, body); + fs.chmodSync(file, 0o755); + return file; +} + +// A lock owned by another live process on this host (the test runner). +function liveForeignLock(lock, time = Date.now()) { + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: process.ppid, host: os.hostname(), time })); +} + +// A stand-in for child_process.spawn: runs `body(command, args)` and exits +// with the code it returns, asynchronously, like a real child. +function fakeSpawn(body) { + const { EventEmitter } = require('node:events'); + return (command, args) => { + const child = new EventEmitter(); + child.exitCode = null; + child.signalCode = null; + child.pid = 0; + setImmediate(() => { + child.exitCode = body(command, args); + child.emit('exit', child.exitCode, null); + }); + return child; + }; +} + +const posixOnly = { skip: process.platform === 'win32' && 'POSIX shell scripts' }; + +describe('package.json', () => { + const pkg = JSON.parse(fs.readFileSync(path.join(PKG_DIR, 'package.json'), 'utf8')); + + test('every bin points at one target, so `npx @deepgram/cli` resolves', () => { + assert.deepEqual(new Set(Object.values(pkg.bin)), new Set(['bin/dg.js'])); + assert.deepEqual(Object.keys(pkg.bin).sort(), ['deepgram', 'dg']); + }); + + test('ships no deepctl bin, dependencies or install scripts', () => { + assert.equal(pkg.bin.deepctl, undefined); + assert.equal(pkg.dependencies, undefined); + for (const hook of ['preinstall', 'install', 'postinstall', 'prepare']) { + assert.equal(pkg.scripts && pkg.scripts[hook], undefined); + } + }); + + test('ships the same LICENSE as the repository', () => { + const repoLicense = path.join(PKG_DIR, '..', 'LICENSE'); + const own = fs.readFileSync(path.join(PKG_DIR, 'LICENSE'), 'utf8'); + assert.match(own, /MIT License/); + if (fs.existsSync(repoLicense)) assert.equal(own, fs.readFileSync(repoLicense, 'utf8')); + }); + + test('version is a plain release version', () => { + assert.match(launcher.packageVersion(PKG_DIR), /^\d+\.\d+\.\d+$/); + }); +}); + +describe('PyPI spec', () => { + test('defaults to the package version', () => { + assert.equal(launcher.pypiSpec({}, '1.2.3'), 'deepctl==1.2.3'); + assert.equal(launcher.venvKey({}, '1.2.3'), '1.2.3'); + }); + + test('DEEPCTL_NPM_PYPI_SPEC overrides it and gets its own venv key', () => { + const env = { DEEPCTL_NPM_PYPI_SPEC: 'deepctl==1.0.0' }; + assert.equal(launcher.pypiSpec(env, '1.2.3'), 'deepctl==1.0.0'); + assert.match(launcher.venvKey(env, '1.2.3'), /^1\.2\.3\+[0-9a-f]{12}$/); + }); +}); + +describe('PATH resolution', () => { + test('skips empty and relative entries', () => { + assert.deepEqual(launcher.pathDirs({ PATH: ':/usr/bin::./bin:bin:/opt/x' }, 'linux'), ['/usr/bin', '/opt/x']); + }); + + test('Windows reads Path, splits on ; and only accepts .exe/.com', () => { + const env = { Path: 'C:\\Tools;;.\\local;D:\\bin' }; + assert.deepEqual(launcher.pathDirs(env, 'win32'), ['C:\\Tools', 'D:\\bin']); + assert.deepEqual(launcher.exeNames('uv', 'win32'), ['uv.exe', 'uv.com']); + const seen = []; + const found = launcher.whichAll('uv', { + env, + platform: 'win32', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + realpath: (f) => f, + isExecutable: (f) => { + seen.push(f); + return f === 'D:\\bin\\uv.exe'; + }, + }); + assert.deepEqual(found, ['D:\\bin\\uv.exe']); + assert.ok(seen.every((f) => /\.(exe|com)$/.test(f))); + }); + + test('finds executables in PATH order', posixOnly, () => { + const dir = tmpdir(); + const a = fakeExe(path.join(dir, 'a'), 'tool'); + const b = fakeExe(path.join(dir, 'b'), 'tool'); + fs.writeFileSync(path.join(dir, 'c-tool'), 'not executable'); + const env = { PATH: [path.join(dir, 'a'), path.join(dir, 'b')].join(':') }; + assert.deepEqual(launcher.whichAll('tool', { env, platform: 'linux' }), [a, b]); + }); + + test('never resolves into the launcher package (recursion guard)', posixOnly, () => { + const dir = tmpdir(); + const pkgDir = path.join(dir, 'node_modules', '@deepgram', 'cli'); + const inside = fakeExe(path.join(pkgDir, 'bin'), 'deepctl'); + const shims = path.join(dir, 'shims'); + fs.mkdirSync(shims); + fs.symlinkSync(inside, path.join(shims, 'deepctl')); + const real = fakeExe(path.join(dir, 'real'), 'deepctl'); + const env = { PATH: [shims, path.join(pkgDir, 'bin'), path.join(dir, 'real')].join(':') }; + assert.deepEqual(launcher.whichAll('deepctl', { env, pkgDir, platform: 'linux' }), [real]); + }); +}); + +describe('uv discovery', () => { + test('prefers PATH, then ~/.local/bin, then ~/.cargo/bin', () => { + const present = new Set(['/home/u/.local/bin/uv', '/home/u/.cargo/bin/uv']); + const opts = { env: { PATH: '/usr/bin' }, platform: 'linux', home: '/home/u', realpath: (f) => f, isExecutable: (f) => present.has(f) }; + assert.equal(launcher.findUv(opts), '/home/u/.local/bin/uv'); + present.add('/usr/bin/uv'); + assert.equal(launcher.findUv(opts), '/usr/bin/uv'); + present.clear(); + assert.equal(launcher.findUv(opts), null); + }); + + test('Windows fallbacks use uv.exe', () => { + const opts = { + env: { Path: 'C:\\Windows' }, + platform: 'win32', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + home: 'C:\\Users\\u', + realpath: (f) => f, + isExecutable: (f) => f === 'C:\\Users\\u\\.cargo\\bin\\uv.exe', + }; + assert.equal(launcher.findUv(opts), 'C:\\Users\\u\\.cargo\\bin\\uv.exe'); + }); +}); + +describe('Python discovery', () => { + test('parses the probe output', () => { + assert.deepEqual(launcher.parseProbeOutput('(3, 12)\n'), [3, 12]); + assert.deepEqual(launcher.parseProbeOutput('(3, 9)\nTraceback...'), [3, 9]); + assert.equal(launcher.parseProbeOutput(''), null); + assert.equal(launcher.parseProbeOutput(undefined), null); + }); + + test('compares versions', () => { + assert.ok(launcher.versionAtLeast([3, 10], [3, 10])); + assert.ok(launcher.versionAtLeast([4, 0], [3, 10])); + assert.ok(!launcher.versionAtLeast([3, 9], [3, 10])); + }); + + test('POSIX probe order, skipping the macOS /usr/bin/python3 stub', () => { + const present = new Set(['/usr/bin/python3', '/usr/local/bin/python3.11', '/usr/local/bin/python3']); + const opts = { env: { PATH: '/usr/local/bin:/usr/bin' }, realpath: (f) => f, isExecutable: (f) => present.has(f) }; + const darwin = launcher.pythonCandidates({ ...opts, platform: 'darwin' }).map((c) => c.command); + assert.deepEqual(darwin, ['/usr/local/bin/python3.11', '/usr/local/bin/python3']); + const linux = launcher.pythonCandidates({ ...opts, platform: 'linux' }).map((c) => c.command); + assert.deepEqual(linux, ['/usr/local/bin/python3.11', '/usr/local/bin/python3', '/usr/bin/python3']); + }); + + test('Windows tries py -3.13..-3.10 and -3, then python, with WindowsApps last', () => { + const present = new Set([ + 'C:\\Windows\\py.exe', + 'C:\\Users\\u\\AppData\\Local\\Microsoft\\WindowsApps\\python.exe', + 'C:\\Python312\\python.exe', + ]); + const opts = { + env: { Path: 'C:\\Windows;C:\\Users\\u\\AppData\\Local\\Microsoft\\WindowsApps;C:\\Python312' }, + platform: 'win32', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + realpath: (f) => f, + isExecutable: (f) => present.has(f), + }; + const got = launcher.pythonCandidates(opts).map((c) => [c.command, ...c.args].join(' ')); + assert.deepEqual(got, [ + 'C:\\Windows\\py.exe -3.13', + 'C:\\Windows\\py.exe -3.12', + 'C:\\Windows\\py.exe -3.11', + 'C:\\Windows\\py.exe -3.10', + 'C:\\Windows\\py.exe -3', + 'C:\\Python312\\python.exe', + // A real Store Python lives here too; the probe rejects the stub. + 'C:\\Users\\u\\AppData\\Local\\Microsoft\\WindowsApps\\python.exe', + ]); + }); + + test('Windows tries a py found only in WindowsApps, after everything else', () => { + const apps = 'C:\\Users\\u\\AppData\\Local\\Microsoft\\WindowsApps'; + const present = new Set([`${apps}\\py.exe`, `${apps}\\python.exe`]); + const opts = { + env: { Path: apps }, + platform: 'win32', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + realpath: (f) => f, + isExecutable: (f) => present.has(f), + }; + const got = launcher.pythonCandidates(opts).map((c) => [c.command, ...c.args].join(' ')); + assert.equal(got.length, 6); + assert.equal(got[0], `${apps}\\py.exe -3.13`); + assert.equal(got[4], `${apps}\\py.exe -3`); + assert.equal(got[5], `${apps}\\python.exe`); + }); + + test('the version probe rejects a Store stub that prints no version', async () => { + const stub = async () => ({ status: 9009, stdout: '' }); + const info = await launcher.probePython({ command: 'python.exe', args: [] }, stub); + assert.equal(info.ok, false); + assert.equal(info.version, null); + }); + + test('findPython returns the first usable one and records the rest', async () => { + const present = new Set(['/usr/bin/python3.12', '/usr/bin/python3']); + const probes = { + '/usr/bin/python3.12': { version: [3, 12], ok: false, hasVenv: false }, + '/usr/bin/python3': { version: [3, 11], ok: true, hasVenv: true }, + }; + const opts = { + env: { PATH: '/usr/bin' }, + platform: 'linux', + realpath: (f) => f, + isExecutable: (f) => present.has(f), + probe: (c) => probes[c.command], + }; + const { python, seen } = await launcher.findPython(opts); + assert.equal(python.command, '/usr/bin/python3'); + assert.equal(seen.length, 2); + }); + + test('the help text names the fix for what was found', () => { + const noVenv = launcher.installHelp([{ version: [3, 12], ok: false, hasVenv: false }]); + assert.match(noVenv, /python3-venv/); + assert.match(noVenv, /astral\.sh\/uv\/install\.sh/); + assert.match(noVenv, /brew install deepgram\/tap\/deepgram/); + assert.match(noVenv, /python\.bat shims and uv\.cmd are skipped/); + const old = launcher.installHelp([{ version: [3, 9], ok: false, hasVenv: true }]); + assert.match(old, /found 3\.9/); + assert.doesNotMatch(old, /python3-venv/); + }); +}); + +describe('cache directory', () => { + test('Linux honours an absolute XDG_CACHE_HOME', () => { + const roots = launcher.cacheRoots({ env: { XDG_CACHE_HOME: '/xdg' }, platform: 'linux', home: '/home/u', tmpdir: '/tmp', uid: 7 }); + assert.deepEqual(roots, ['/xdg/deepgram-cli/venvs', '/tmp/deepgram-cli-7/venvs']); + const relative = launcher.cacheRoots({ env: { XDG_CACHE_HOME: 'rel' }, platform: 'linux', home: '/home/u', tmpdir: '/tmp', uid: 7 }); + assert.equal(relative[0], '/home/u/.cache/deepgram-cli/venvs'); + }); + + test('macOS uses ~/Library/Caches', () => { + const roots = launcher.cacheRoots({ env: {}, platform: 'darwin', home: '/Users/u', tmpdir: '/tmp', uid: 501 }); + assert.equal(roots[0], '/Users/u/Library/Caches/deepgram-cli/venvs'); + }); + + test('Windows uses LOCALAPPDATA only, with no %TEMP% fallback', () => { + const roots = launcher.cacheRoots({ env: { LOCALAPPDATA: 'C:\\Users\\u\\AppData\\Local' }, platform: 'win32', home: 'C:\\Users\\u', tmpdir: 'C:\\Temp', uid: 'user' }); + assert.deepEqual(roots, ['C:\\Users\\u\\AppData\\Local\\deepgram-cli\\venvs']); + // A relative LOCALAPPDATA is ignored, like a relative XDG_CACHE_HOME. + const relative = launcher.cacheRoots({ env: { LOCALAPPDATA: 'AppData' }, platform: 'win32', home: 'C:\\Users\\u' }); + assert.deepEqual(relative, ['C:\\Users\\u\\AppData\\Local\\deepgram-cli\\venvs']); + const layout = launcher.venvLayout('C:\\v', 'win32'); + assert.equal(layout.deepctl, 'C:\\v\\Scripts\\deepctl.exe'); + }); + + test('a cached venv is used only when complete, from either root', () => { + const opts = { env: {}, platform: 'linux', home: '/home/u', tmpdir: '/tmp', uid: 1, trustRoot: () => true }; + const files = new Set(['/tmp/deepgram-cli-1/venvs/1.0.0/.complete', '/tmp/deepgram-cli-1/venvs/1.0.0/bin/deepctl', '/tmp/deepgram-cli-1/venvs/1.0.0/bin/python']); + const hit = launcher.findCachedVenv('1.0.0', { ...opts, exists: (f) => files.has(f) }); + assert.equal(hit.deepctl, '/tmp/deepgram-cli-1/venvs/1.0.0/bin/deepctl'); + files.delete('/tmp/deepgram-cli-1/venvs/1.0.0/.complete'); + assert.equal(launcher.findCachedVenv('1.0.0', { ...opts, exists: (f) => files.has(f) }), null); + }); + + test('a venv whose python no longer resolves is not used', () => { + const opts = { env: {}, platform: 'linux', home: '/home/u', tmpdir: '/tmp', uid: 1, trustRoot: () => true }; + const files = new Set(['/home/u/.cache/deepgram-cli/venvs/1.0.0/.complete', '/home/u/.cache/deepgram-cli/venvs/1.0.0/bin/deepctl']); + assert.equal(launcher.findCachedVenv('1.0.0', { ...opts, exists: (f) => files.has(f) }), null); + }); + + test('a dangling bin/python symlink makes a venv unusable (real filesystem)', posixOnly, () => { + const dir = tmpdir(); + const layout = launcher.venvLayout(dir, 'linux'); + fs.mkdirSync(path.dirname(layout.python), { recursive: true }); + fs.writeFileSync(layout.marker, ''); + fakeExe(path.dirname(layout.deepctl), 'deepctl'); + fs.symlinkSync('/nonexistent/python3', layout.python); + assert.equal(launcher.venvUsable(layout), false); + fs.unlinkSync(layout.python); + fs.symlinkSync(process.execPath, layout.python); + assert.equal(launcher.venvUsable(layout), true); + }); + + test('an untrusted root is never used, however complete its venv looks', () => { + const opts = { env: {}, platform: 'linux', home: '/home/u', tmpdir: '/tmp', uid: 1, trustRoot: (spec) => !spec.shared }; + const files = new Set(['/tmp/deepgram-cli-1/venvs/1.0.0/.complete', '/tmp/deepgram-cli-1/venvs/1.0.0/bin/deepctl']); + assert.equal(launcher.findCachedVenv('1.0.0', { ...opts, exists: (f) => files.has(f) }), null); + }); +}); + +describe('Windows never reuses a cached venv', () => { + const win = { + env: { Path: 'C:\\bin', LOCALAPPDATA: 'C:\\Users\\u\\AppData\\Local' }, + platform: 'win32', + home: 'C:\\Users\\u', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + realpath: (f) => f, + // Every file "exists": a complete-looking venv sits in the cache. + exists: () => true, + }; + + test('cache reuse is off on win32 and on elsewhere', () => { + assert.equal(launcher.cacheReuse({ platform: 'win32' }), false); + assert.equal(launcher.cacheReuse({ platform: 'linux' }), true); + assert.equal(launcher.cacheReuse({ platform: 'darwin' }), true); + }); + + test('a complete-looking venv is ignored, even with a permissive trust check', () => { + assert.equal(launcher.findCachedVenv('1.0.0', win), null); + assert.equal(launcher.findCachedVenv('1.0.0', { ...win, trustRoot: () => true }), null); + }); + + test('resolveRunner goes to uv instead of the cached venv', async () => { + const r = await launcher.resolveRunner(['--version'], { ...win, version: '1.0.0', isExecutable: (f) => f === 'C:\\bin\\uv.exe' }); + assert.equal(r.strategy, 'uvx'); + assert.equal(r.command, 'C:\\bin\\uv.exe'); + }); + + test('without uv, the venv is built fresh for this run and marked for cleanup', posixOnly, async () => { + // The win32 path logic is exercised on a POSIX filesystem by turning + // reuse off directly; Windows CI runs the real thing (venv smoke test). + const home = tmpdir(); + const root = path.join(home, 'c', 'deepgram-cli', 'venvs'); + fs.mkdirSync(path.join(root, '1.0.0', 'bin'), { recursive: true }); + const leaked = path.join(root, '1.0.0-Old123'); + fs.mkdirSync(leaked); + const old = new Date(Date.now() - 2 * 60 * 60 * 1000); + fs.utimesSync(leaked, old, old); + for (const f of ['.complete', 'bin/deepctl', 'bin/python']) fs.writeFileSync(path.join(root, '1.0.0', f), ''); + const calls = []; + const spawn = fakeSpawn((command, args) => { + calls.push(args.slice(-2).join(' ')); + if (args.includes('venv')) fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + return 0; + }); + const lines = []; + const r = await launcher.resolveRunner(['x'], { + env: { PATH: '/usr/bin', XDG_CACHE_HOME: path.join(home, 'c') }, + version: '1.0.0', + platform: 'linux', + cacheReuse: false, + probe: () => ({ version: [3, 12], ok: true, hasVenv: true }), + isExecutable: (f) => f === '/usr/bin/python3', + realpath: (f) => f, + spawn, + stderr: { write: (t) => lines.push(t) }, + }); + // Not the planted 1.0.0 venv: a new, unpredictably named sibling. + assert.equal(path.dirname(r.cleanup), root); + assert.match(path.basename(r.cleanup), /^1\.0\.0-.{6}$/); + assert.equal(r.command, path.join(r.cleanup, 'bin', 'deepctl')); + assert.equal(calls.length, 3); + assert.match(lines.join(''), /for this run.*install uv.*install\.ps1/); + assert.equal(lines.join('').trimEnd().split('\n').length, 1, 'the hint is one line'); + assert.equal(fs.existsSync(path.join(r.cleanup, '.complete')), false); + assert.equal(fs.existsSync(leaked), false, 'a leaked one-run venv was pruned'); + }); +}); + +describe('pruning one-run venvs', () => { + const HOUR = 60 * 60 * 1000; + function aged(root, name, ageMs) { + const dir = path.join(root, name); + fs.mkdirSync(path.join(dir, 'Scripts'), { recursive: true }); + const t = new Date(Date.now() - ageMs); + fs.utimesSync(dir, t, t); + return dir; + } + + test('removes leaked one-run venvs over an hour old, of any version, and keeps the rest', () => { + const root = tmpdir(); + const leaked = aged(root, '1.0.0-AbC123', 2 * HOUR); + const olderVersion = aged(root, '0.9.0-xyz789', 3 * 24 * HOUR); + const halfDeleted = aged(root, '1.0.0-Qq11Ww.del', 2 * HOUR); + const concurrent = aged(root, '1.0.0-Zz99Yy', 5 * 60 * 1000); + const unrelated = aged(root, 'notes', 2 * HOUR); + launcher.pruneOneRunVenvs(root); + assert.equal(fs.existsSync(leaked), false); + assert.equal(fs.existsSync(`${leaked}.del`), false); + assert.equal(fs.existsSync(olderVersion), false); + assert.equal(fs.existsSync(halfDeleted), false); + assert.equal(fs.existsSync(concurrent), true, 'a run still setting up keeps its venv'); + assert.equal(fs.existsSync(unrelated), true); + }); + + test('a venv that cannot be moved aside (in use on Windows) is left alone', () => { + const root = tmpdir(); + const busy = aged(root, '1.0.0-AbC123', 2 * HOUR); + const realRename = fs.renameSync; + fs.renameSync = () => { + throw Object.assign(new Error('EBUSY'), { code: 'EBUSY' }); + }; + try { + launcher.pruneOneRunVenvs(root); + } finally { + fs.renameSync = realRename; + } + assert.equal(fs.existsSync(busy), true); + }); + + test('a missing directory is not an error', () => { + assert.doesNotThrow(() => launcher.pruneOneRunVenvs(path.join(tmpdir(), 'nope'))); + }); +}); + +describe('cache root ownership (POSIX)', () => { + const dirStat = (uid, mode, isDir = true, gid = uid) => ({ uid, gid, mode: (isDir ? 0o040000 : 0o120000) | mode, isDirectory: () => isDir, isSymbolicLink: () => !isDir }); + const shared = { dir: '/tmp/deepgram-cli-1000/venvs', shared: true }; + const home = { dir: '/home/u/.cache/deepgram-cli/venvs', shared: false }; + // Sane ancestors; individual tests override entries. + const ancestors = { + '/': dirStat(0, 0o755), + '/tmp': dirStat(0, 0o1777), + '/home': dirStat(0, 0o755), + '/home/u': dirStat(1000, 0o755), + '/home/u/.cache': dirStat(1000, 0o700), + }; + const check = (spec, stats, realpath = (d) => d, gid = 1000) => { + const all = { ...ancestors, ...stats }; + return launcher.rootTrusted(spec, { platform: 'linux', uid: 1000, gid, realpath, lstat: (d) => { + if (!(d in all)) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + return all[d]; + } }); + }; + + test('a private directory we own under sticky /tmp is trusted', () => { + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(1000, 0o700), '/tmp/deepgram-cli-1000/venvs': dirStat(1000, 0o700) }), true); + }); + + test('a shared root owned by another user is rejected', () => { + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(2000, 0o700), '/tmp/deepgram-cli-1000/venvs': dirStat(1000, 0o700) }), false); + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(1000, 0o700), '/tmp/deepgram-cli-1000/venvs': dirStat(2000, 0o700) }), false); + }); + + test('a shared root others can read or write is rejected, even if ours', () => { + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(1000, 0o777), '/tmp/deepgram-cli-1000/venvs': dirStat(1000, 0o700) }), false); + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(1000, 0o750), '/tmp/deepgram-cli-1000/venvs': dirStat(1000, 0o700) }), false); + }); + + test('a symlinked root is rejected', () => { + assert.equal(check(shared, { '/tmp/deepgram-cli-1000': dirStat(1000, 0o777, false), '/tmp/deepgram-cli-1000/venvs': dirStat(1000, 0o700) }), false); + }); + + test('the home root may be readable by others but not writable', () => { + const base = '/home/u/.cache/deepgram-cli'; + assert.equal(check(home, { [base]: dirStat(1000, 0o755), [`${base}/venvs`]: dirStat(1000, 0o755) }), true); + assert.equal(check(home, { [base]: dirStat(1000, 0o775), [`${base}/venvs`]: dirStat(1000, 0o755) }), false); + assert.equal(check(home, { [base]: dirStat(2000, 0o755), [`${base}/venvs`]: dirStat(1000, 0o755) }), false); + }); + + test('an ancestor another user can write to is rejected', () => { + const base = '/home/u/.cache/deepgram-cli'; + const ok = { [base]: dirStat(1000, 0o700), [`${base}/venvs`]: dirStat(1000, 0o700) }; + assert.equal(check(home, ok), true); + // HOME=/tmp style: ~/.cache made world-writable by someone else. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(2000, 0o777) }), false); + // Ours but group-writable by a shared group, not sticky: swappable by the group. + assert.equal(check(home, { ...ok, '/home/u': dirStat(1000, 0o775, true, 50) }), false); + // World-writable but sticky and root-owned, like /tmp: fine. + assert.equal(check(home, { ...ok, '/home': dirStat(0, 0o1777) }), true); + // Owned by a third user, even read-only: rejected. + assert.equal(check(home, { ...ok, '/home': dirStat(2000, 0o755) }), false); + }); + + test('a root-owned symlink ancestor is allowed, an attacker-owned one is not', () => { + const base = '/home/u/.cache/deepgram-cli'; + const ok = { [base]: dirStat(1000, 0o700), [`${base}/venvs`]: dirStat(1000, 0o700) }; + // /home -> /data/home, as /var -> /private/var on macOS. + const real = (d) => d.replace(/^\/home/, '/data/home'); + const target = { '/data': dirStat(0, 0o755), '/data/home': dirStat(0, 0o755), '/data/home/u': dirStat(1000, 0o755), '/data/home/u/.cache': dirStat(1000, 0o700) }; + assert.equal(check(home, { ...ok, ...target, '/home': dirStat(0, 0o777, false) }, real), true); + assert.equal(check(home, { ...ok, ...target, '/home': dirStat(2000, 0o777, false) }, real), false); + // The resolved path is checked too. + assert.equal(check(home, { ...ok, ...target, '/data/home': dirStat(2000, 0o777), '/home': dirStat(0, 0o777, false) }, real), false); + }); + + test('a missing root is not trusted', () => { + assert.equal(check(shared, {}), false); + }); + + test('on a real filesystem, a 0777 directory we own is rejected and 0700 accepted', posixOnly, () => { + const base = tmpdir(); + const spec = { dir: path.join(base, 'deepgram-cli-x', 'venvs'), shared: true }; + fs.mkdirSync(spec.dir, { recursive: true }); + fs.chmodSync(path.dirname(spec.dir), 0o777); + fs.chmodSync(spec.dir, 0o700); + assert.equal(launcher.rootTrusted(spec), false); + fs.chmodSync(path.dirname(spec.dir), 0o700); + assert.equal(launcher.rootTrusted(spec), true); + const link = { dir: path.join(base, 'linked', 'venvs'), shared: true }; + fs.symlinkSync(path.dirname(spec.dir), path.join(base, 'linked')); + assert.equal(launcher.rootTrusted(link), false); + }); + + test('our own group-writable ~/.cache in our private group is accepted (umask 002)', () => { + const base = '/home/u/.cache/deepgram-cli'; + const ok = { [base]: dirStat(1000, 0o700), [`${base}/venvs`]: dirStat(1000, 0o700) }; + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775) }), true); + assert.equal(check(home, { ...ok, '/home/u': dirStat(1000, 0o775) }), true); + // Group-writable with a shared group (staff, users): rejected. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 50) }), false); + // Writable by everyone: rejected even when ours. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o777) }), false); + // Group-writable, our gid, but owned by someone else: rejected. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(2000, 0o775, true, 1000) }), false); + }); + + test('a group-writable ~/.cache in a shared primary group is rejected even though it is ours', () => { + // SUSE / Debian USERGROUPS=no: primary group `users` (100), shared with + // every other user, so gid matches ours but is not a private group. + const base = '/home/u/.cache/deepgram-cli'; + const ok = { [base]: dirStat(1000, 0o700, true, 100), [`${base}/venvs`]: dirStat(1000, 0o700, true, 100) }; + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 100) }, (d) => d, 100), false); + // The same directory without group write is fine. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o755, true, 100) }, (d) => d, 100), true); + }); + + test('prepareRoot creates missing parents 0700 even under umask 002', posixOnly, () => { + const home = tmpdir(); + const spec = { dir: path.join(home, '.cache', 'deepgram-cli', 'venvs'), shared: false }; + const old = process.umask(0o002); + try { + assert.equal(launcher.prepareRoot(spec), true); + } finally { + process.umask(old); + } + assert.equal(fs.statSync(path.join(home, '.cache')).mode & 0o777, 0o700); + assert.equal(fs.statSync(spec.dir).mode & 0o777, 0o700); + }); + + test('no Windows root is trusted: ACLs and reparse points cannot be checked', () => { + assert.equal(launcher.rootTrusted({ dir: 'C:\\x\\venvs', shared: false }, { platform: 'win32' }), false); + assert.equal(launcher.rootTrusted({ dir: 'C:\\x\\venvs', shared: true }, { platform: 'win32' }), false); + }); +}); + +describe('pruning other versions', () => { + const DAY = 24 * 60 * 60 * 1000; + function venv(root, name, { markerAgeDays, dirAgeMs } = {}) { + const dir = path.join(root, name); + fs.mkdirSync(path.join(dir, 'bin'), { recursive: true }); + if (markerAgeDays !== undefined) { + const marker = path.join(dir, '.complete'); + fs.writeFileSync(marker, ''); + const t = new Date(Date.now() - markerAgeDays * DAY); + fs.utimesSync(marker, t, t); + } + if (dirAgeMs !== undefined) { + const t = new Date(Date.now() - dirAgeMs); + fs.utimesSync(dir, t, t); + } + return dir; + } + + test('keeps recently used versions and removes long-idle ones', () => { + const root = tmpdir(); + venv(root, '1.0.0', { markerAgeDays: 0 }); + const recent = venv(root, '0.9.0', { markerAgeDays: 2 }); + const idle = venv(root, '0.8.0', { markerAgeDays: 45 }); + launcher.pruneSiblings(root, '1.0.0'); + assert.equal(fs.existsSync(recent), true); + assert.equal(fs.existsSync(idle), false); + }); + + test('a half-built venv is removed only once no build could still own it', () => { + const root = tmpdir(); + const fresh = venv(root, '0.9.0', { dirAgeMs: 60 * 1000 }); + const dead = venv(root, '0.8.0', { dirAgeMs: 60 * 60 * 1000 }); + launcher.pruneSiblings(root, '1.0.0'); + assert.equal(fs.existsSync(fresh), true); + assert.equal(fs.existsSync(dead), false); + }); + + test('never removes a version with a live lock', () => { + const root = tmpdir(); + const locked = venv(root, '0.8.0', { markerAgeDays: 90 }); + liveForeignLock(`${locked}.lock`); + launcher.pruneSiblings(root, '1.0.0'); + assert.equal(fs.existsSync(locked), true); + }); +}); + +describe('lock', () => { + test('acquire, then release', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + const release = await launcher.acquireLock(lock); + const owner = JSON.parse(fs.readFileSync(path.join(lock, 'owner.json'), 'utf8')); + assert.equal(owner.pid, process.pid); + release(); + assert.equal(fs.existsSync(lock), false); + }); + + test('a lock held by a live process times out and announces the wait once', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + liveForeignLock(lock); + let waits = 0; + await assert.rejects( + launcher.acquireLock(lock, { waitMs: 300, pollMs: 50, onWait: () => waits++ }), + /Timed out waiting/, + ); + assert.equal(waits, 1); + }); + + test('a lock whose pid is dead is stale and is taken over', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: 999999, host: os.hostname(), time: Date.now() })); + assert.equal(launcher.lockIsStale(lock, Date.now(), () => false), true); + assert.equal(launcher.lockIsStale(lock, Date.now(), () => true), false); + const release = await launcher.acquireLock(lock, { waitMs: 0 }); + release(); + }); + + test('a same-host lock with a live pid is kept for up to an hour', () => { + const lock = path.join(tmpdir(), 'v.lock'); + liveForeignLock(lock, Date.now() - 30 * 60 * 1000); + assert.equal(launcher.lockIsStale(lock), false); + assert.equal(launcher.lockIsStale(lock, Date.now() + 31 * 60 * 1000), true); + }); + + test('a lock naming our own pid is a reused pid and stale (crashed container restart)', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: process.pid, host: os.hostname(), time: Date.now() })); + assert.equal(launcher.lockIsStale(lock), true); + const release = await launcher.acquireLock(lock, { waitMs: 0 }); + release(); + assert.equal(fs.existsSync(lock), false); + }); + + test('breaking a stale lock removes it only if it is still the judged owner', () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: 999999, host: os.hostname(), time: 1 })); + launcher.breakStaleLock(lock); + assert.equal(fs.existsSync(lock), false); + assert.deepEqual(fs.readdirSync(path.dirname(lock)), []); + }); + + test('a fresh lock moved aside by a racing waiter is put back', () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: 1, host: 'h', time: 1 })); + // Simulate the race: the owner changes between judging and renaming. + const realRename = fs.renameSync; + let swapped = false; + fs.renameSync = (from, to) => { + if (!swapped) { + swapped = true; + fs.writeFileSync(path.join(from, 'owner.json'), JSON.stringify({ pid: 2, host: 'h', time: 2 })); + } + return realRename(from, to); + }; + try { + launcher.breakStaleLock(lock); + } finally { + fs.renameSync = realRename; + } + assert.equal(JSON.parse(fs.readFileSync(path.join(lock, 'owner.json'), 'utf8')).pid, 2); + }); + + test('a lock from another host is judged by age, not local pids', () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: 999999, host: 'elsewhere', time: Date.now() })); + assert.equal(launcher.lockIsStale(lock, Date.now(), () => false), false); + assert.equal(launcher.lockIsStale(lock, Date.now() + 11 * 60 * 1000, () => false), true); + }); + + test('owner.json is replaced whole, with no temp file left behind', () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + launcher.writeOwner(lock, { pid: 1, host: 'h', time: 1 }); + launcher.writeOwner(lock, { pid: 1, host: 'h', time: 1, pgid: 42 }); + assert.deepEqual(fs.readdirSync(lock), ['owner.json']); + assert.equal(launcher.readOwner(lock).owner.pgid, 42); + }); + + test('a torn or missing owner.json reads as no owner, and age decides', () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + assert.equal(launcher.readOwner(lock).owner, null); + fs.writeFileSync(path.join(lock, 'owner.json'), '{"pid": 12'); + const read = launcher.readOwner(lock); + assert.equal(read.owner, null); + assert.equal(read.raw, '{"pid": 12'); + // Fresh directory: not stale, however unreadable its owner. + assert.equal(launcher.lockIsStale(lock), false); + assert.equal(launcher.lockIsStale(lock, Date.now() + 11 * 60 * 1000), true); + }); + + test('a waiter gives a not-yet-written owner.json a moment before judging by age', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + fs.mkdirSync(lock); + // Old directory, so an immediate age judgement would call it stale. + const old = new Date(Date.now() - 11 * 60 * 1000); + fs.utimesSync(lock, old, old); + setTimeout(() => launcher.writeOwner(lock, { pid: process.ppid, host: os.hostname(), time: Date.now() }), 30); + await assert.rejects(launcher.acquireLock(lock, { waitMs: 200, pollMs: 50 }), /Timed out waiting/); + assert.equal(launcher.readOwner(lock).owner.pid, process.ppid, 'the live owner\'s lock survived'); + }); + + test('release leaves a lock that another process has taken over', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + const release = await launcher.acquireLock(lock); + fs.writeFileSync(path.join(lock, 'owner.json'), JSON.stringify({ pid: 4242, host: os.hostname(), time: 1 })); + release(); + assert.equal(fs.existsSync(lock), true); + }); + + test('a failure writing the lock is a clean LauncherError and leaves no lock', async () => { + const lock = path.join(tmpdir(), 'v.lock'); + const writeFile = () => { + throw Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' }); + }; + await assert.rejects( + launcher.acquireLock(lock, { writeFile }), + (err) => err instanceof launcher.LauncherError && /Could not create the setup lock.*ENOSPC/.test(err.message), + ); + assert.equal(fs.existsSync(lock), false); + }); +}); + +describe('launcher messages on stderr', () => { + const err = (code) => Object.assign(new Error(code), { code }); + + test('partial writes are completed', () => { + const chunks = []; + const writeSync = (fd, buf, offset, length) => { + const n = Math.min(3, length); + chunks.push(buf.subarray(offset, offset + n).toString()); + return n; + }; + launcher.writeStderr('hello world\n', { writeSync }); + assert.equal(chunks.join(''), 'hello world\n'); + }); + + test('EAGAIN is retried after a short sleep', () => { + let calls = 0; + let sleeps = 0; + const writeSync = (fd, buf, offset, length) => { + if (calls++ < 3) throw err('EAGAIN'); + return length; + }; + launcher.writeStderr('x', { writeSync, sleep: () => sleeps++ }); + assert.equal(calls, 4); + assert.equal(sleeps, 3); + }); + + test('a stderr that stays full is given up on, not thrown', () => { + let calls = 0; + const writeSync = () => { + calls++; + throw err('EAGAIN'); + }; + assert.doesNotThrow(() => launcher.writeStderr('x', { writeSync, sleep: () => {} })); + assert.equal(calls, 101); + }); + + test('EPIPE and other errors are swallowed', () => { + for (const code of ['EPIPE', 'EBADF', 'EIO']) { + assert.doesNotThrow(() => launcher.writeStderr('x', { writeSync: () => { throw err(code); } })); + } + }); +}); + +describe('child environment', () => { + test('marks the launcher and adds nothing else', () => { + const env = launcher.childEnv({ HOME: '/h' }, 'uvx'); + assert.deepEqual(env, { HOME: '/h', DEEPCTL_LAUNCHER: 'npm' }); + assert.equal(launcher.childEnv({}, 'venv').DEEPCTL_LAUNCHER, 'npm'); + }); + + test('a package unpacked under npm\'s _npx cache is reported as npx', () => { + const npx = '/home/u/.npm/_npx/0123abcd/node_modules/@deepgram/cli'; + assert.equal(launcher.childEnv({}, 'uvx', npx).DEEPCTL_LAUNCHER, 'npx'); + assert.equal(launcher.launcherKind('C:\\Users\\u\\AppData\\Local\\npm-cache\\_npx\\ab\\node_modules\\@deepgram\\cli'), 'npx'); + // How the parent was started does not matter: a global dg is npm. + assert.equal(launcher.childEnv({ npm_command: 'exec' }, 'venv', '/usr/local/lib/node_modules/@deepgram/cli').DEEPCTL_LAUNCHER, 'npm'); + }); + + test('the path strategy removes DEEPCTL_LAUNCHER', () => { + const env = launcher.childEnv({ DEEPCTL_LAUNCHER: 'npm' }, 'path'); + assert.equal('DEEPCTL_LAUNCHER' in env, false); + }); +}); + +describe('exit mapping', () => { + test('passes exit codes through', () => { + assert.deepEqual(launcher.exitAction(0, null, 'linux'), { exit: 0 }); + assert.deepEqual(launcher.exitAction(2, null, 'linux'), { exit: 2 }); + }); + + test('re-raises signals on POSIX with a 128+n fallback', () => { + assert.deepEqual(launcher.exitAction(null, 'SIGTERM', 'linux'), { signal: 'SIGTERM', exit: 128 + os.constants.signals.SIGTERM }); + assert.deepEqual(launcher.exitAction(null, 'SIGINT', 'darwin'), { signal: 'SIGINT', exit: 128 + os.constants.signals.SIGINT }); + }); + + test('Windows has no signal to re-raise', () => { + assert.deepEqual(launcher.exitAction(null, 'SIGTERM', 'win32'), { exit: 1 }); + }); +}); + +describe('resolveRunner', () => { + test('rejects an unknown DEEPCTL_NPM_RUNNER', async () => { + await assert.rejects(launcher.resolveRunner([], { env: { DEEPCTL_NPM_RUNNER: 'pipx' }, version: '1.0.0' }), /must be one of/); + }); + + test('uses uv with the pinned spec when no venv is cached', async () => { + const r = await launcher.resolveRunner(['--version'], { + env: { PATH: '/bin' }, + version: '1.2.3', + platform: 'linux', + home: '/nonexistent', + tmpdir: '/nonexistent', + realpath: (f) => f, + isExecutable: (f) => f === '/bin/uv', + exists: () => false, + }); + assert.deepEqual(r, { command: '/bin/uv', args: ['tool', 'run', '--from', 'deepctl==1.2.3', 'deepctl', '--version'], strategy: 'uvx' }); + }); + + test('a cached venv wins over uv', async () => { + const r = await launcher.resolveRunner(['x'], { + env: { PATH: '/bin' }, + version: '1.2.3', + platform: 'linux', + home: '/h', + tmpdir: '/t', + uid: 0, + realpath: (f) => f, + isExecutable: (f) => f === '/bin/uv', + exists: (f) => f.startsWith('/h/.cache/deepgram-cli/venvs/1.2.3/'), + trustRoot: () => true, + }); + assert.equal(r.strategy, 'venv'); + assert.equal(r.command, '/h/.cache/deepgram-cli/venvs/1.2.3/bin/deepctl'); + }); + + test('DEEPCTL_NPM_RUNNER=path requires a real deepctl', async () => { + const base = { version: '1.0.0', platform: 'linux', realpath: (f) => f }; + await assert.rejects( + launcher.resolveRunner([], { ...base, env: { PATH: '/bin', DEEPCTL_NPM_RUNNER: 'path' }, isExecutable: () => false }), + /no deepctl was found/, + ); + const r = await launcher.resolveRunner(['a'], { + ...base, + env: { PATH: '/bin', DEEPCTL_NPM_RUNNER: 'path' }, + isExecutable: (f) => f === '/bin/deepctl', + }); + assert.deepEqual(r, { command: '/bin/deepctl', args: ['a'], strategy: 'path' }); + }); + + test('DEEPCTL_NPM_RUNNER=uvx without uv is an error', async () => { + await assert.rejects( + launcher.resolveRunner([], { + env: { PATH: '/bin', DEEPCTL_NPM_RUNNER: 'uvx' }, + version: '1.0.0', + platform: 'linux', + home: '/nonexistent', + realpath: (f) => f, + isExecutable: () => false, + }), + /uv was not found/, + ); + }); +}); + +describe('bin/dg.js end to end', posixOnly, () => { + function runBin(env, args = []) { + return spawnSync(process.execPath, [BIN, ...args], { env, encoding: 'utf8' }); + } + + test('a dg run from inside deepctl (nested launcher) still works', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + fakeExe(bin, 'uv', '#!/bin/sh\necho nested-ok\n'); + const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_LAUNCHER_DEPTH: '1' }); + assert.equal(r.status, 0); + assert.equal(r.stdout, 'nested-ok\n'); + }); + + test('no uv and no Python: clear error on stderr, nothing on stdout, exit 1', () => { + const home = tmpdir(); + const r = runBin({ PATH: '/nonexistent', HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + assert.equal(r.status, 1); + assert.equal(r.stdout, ''); + assert.match(r.stderr, /needs uv or Python 3\.10\+/); + }); + + test('passes arguments, env and exit code through a fake uv', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + fakeExe(bin, 'uv', '#!/bin/sh\necho "$DEEPCTL_LAUNCHER $*"\nexit 7\n'); + const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }, ['listen', 'a b.wav']); + assert.equal(r.status, 7); + const version = launcher.packageVersion(PKG_DIR); + assert.equal(r.stdout, `npm tool run --from deepctl==${version} deepctl listen a b.wav\n`); + }); + + test('a SIGINT sent only to the launcher reaches the child after the grace period', async () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + fakeExe(bin, 'deepctl', '#!/bin/sh\ntrap "echo got-INT; exit 130" INT\nwhile :; do sleep 0.1; done\n'); + const { spawn } = require('node:child_process'); + const child = spawn(process.execPath, [BIN], { + env: { PATH: `${bin}:/usr/bin:/bin`, HOME: home, DEEPCTL_NPM_RUNNER: 'path' }, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + let out = ''; + child.stdout.on('data', (d) => (out += d)); + await new Promise((r) => setTimeout(r, 500)); + const sentAt = Date.now(); + process.kill(child.pid, 'SIGINT'); + const code = await new Promise((r) => child.on('exit', (c) => r(c))); + assert.equal(code, 130); + assert.match(out, /got-INT/); + assert.ok(Date.now() - sentAt >= 900, 'forwarded only after the grace period'); + }); + + // Regression: reading process.stdin.isTTY made Node set O_NONBLOCK on a + // piped fd 0, which deepctl inherited; a reader that had to wait for + // input (an MCP host, `ffmpeg | dg listen`) then failed. + const python3 = ['/usr/bin/python3', '/usr/local/bin/python3', '/opt/homebrew/bin/python3'].find((f) => fs.existsSync(f)); + test('fds 0-2 stay blocking for deepctl, and a delayed stdin writer works', { skip: !python3 && 'needs python3' }, async () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const checker = [ + `#!${python3}`, + 'import fcntl, os, sys', + 'flags = [bool(fcntl.fcntl(fd, fcntl.F_GETFL) & os.O_NONBLOCK) for fd in (0, 1, 2)]', + 'data = sys.stdin.read()', + 'print("nonblock=%s read=%r" % (flags, data))', + '', + ].join('\n'); + // A fake python so the venv strategy runs (and writes its setup line + // to stderr) without touching PyPI: `-m venv` lays down the checker as + // deepctl, everything else succeeds. It's installed under every name + // the launcher probes, or a real python3.12 later on PATH (as on + // GitHub's ubuntu runners) wins and builds a real venv. + const fakePython = [ + '#!/bin/sh', + 'case "$1 $2" in', + ' "-c "*) echo "(3, 12)"; exit 0;;', + ` "-m venv") mkdir -p "$3/bin" && cp "${path.join(home, 'checker')}" "$3/bin/deepctl" && printf '#!/bin/sh\\nexit 0\\n' > "$3/bin/python" && chmod +x "$3/bin/deepctl" "$3/bin/python"; exit 0;;`, + 'esac', + 'exit 0', + '', + ].join('\n'); + for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) { + fakeExe(bin, name, fakePython); + } + fs.writeFileSync(path.join(home, 'checker'), checker); + const { spawn } = require('node:child_process'); + const child = spawn(process.execPath, [BIN], { + env: { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + let out = ''; + let err = ''; + child.stdout.on('data', (d) => (out += d)); + child.stderr.on('data', (d) => (err += d)); + setTimeout(() => child.stdin.end('hello\n'), 1000); + const code = await new Promise((r) => child.on('exit', (c) => r(c))); + assert.equal(code, 0, err); + assert.match(err, /Setting up Deepgram CLI/); + assert.equal(out.trim(), "nonblock=[False, False, False] read='hello\\n'"); + }); + + test('re-raises the signal the child died from', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + fakeExe(bin, 'uv', '#!/bin/sh\nkill -TERM $$\n'); + const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + assert.equal(r.status, null); + assert.equal(r.signal, 'SIGTERM'); + assert.doesNotMatch(r.stderr, /crashed/); + }); + + test('names the signal when the child crashes', { skip: process.platform === 'win32' && 'POSIX signals' }, () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + fakeExe(bin, 'uv', '#!/bin/sh\nkill -SEGV $$\n'); + const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + assert.equal(r.signal, 'SIGSEGV'); + assert.match(r.stderr, /dg: .*uv crashed \(SIGSEGV\)\./); + }); + + test('a venv build step killed by a signal says so', { skip: process.platform === 'win32' && 'POSIX signals' }, () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const fakePython = '#!/bin/sh\ncase "$1 $2" in\n "-c "*) echo "(3, 12)"; exit 0;;\nesac\nkill -KILL $$\n'; + for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) fakeExe(bin, name, fakePython); + const r = runBin({ PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }); + assert.equal(r.status, 1); + assert.match(r.stderr, /failed \(killed by SIGKILL\)/); + }); +}); + +// Regression: signal handlers used to go in only after the synchronous +// first-run setup, so a supervisor's SIGTERM during setup killed the +// launcher and orphaned the setup child (reparented to PID 1), leaving the +// lock and a half-built venv behind. Each step below hangs in a fake Python +// until the test signals the launcher. +describe('a signal during first-run setup', posixOnly, () => { + const version = launcher.packageVersion(PKG_DIR); + + // Dead, or a zombie nobody has reaped yet (PID 1 in a container). + function gone(pid) { + try { + process.kill(pid, 0); + } catch { + return true; + } + try { + return /^\d+ \(.*\) Z/.test(fs.readFileSync(`/proc/${pid}/stat`, 'utf8')); + } catch { + return false; + } + } + + async function waitFor(predicate, what, ms = 15000) { + const deadline = Date.now() + ms; + while (!predicate()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((r) => setTimeout(r, 25)); + } + } + + // Waits until the lock's owner.json names `pgid` as the setup group. + function waitForGroup(root, pgid) { + const file = path.join(root, `${version}.lock`, 'owner.json'); + return waitFor(() => { + const { owner } = launcher.readOwner(path.dirname(file)); + return Boolean(owner) && owner.pgid === pgid; + }, `owner.json to record pgid ${pgid}`); + } + + function workspace(hangAt) { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const pidfile = path.join(home, 'pids'); + // `hang` leaves a grandchild (sleep) under the step's shell, the way + // pip leaves build subprocesses, and records both pids. + const fakePython = [ + '#!/bin/sh', + `hang() { sleep 300 & echo "$$ $!" > "${pidfile}.tmp"; mv "${pidfile}.tmp" "${pidfile}"; wait; exit 0; }`, + 'case "$1 $2" in', + ' "-c "*) echo "(3, 12)"; exit 0;;', + ' "-m venv") mkdir -p "$3/bin" && cp "$0" "$3/bin/python" && printf \'#!/bin/sh\\nexit 0\\n\' > "$3/bin/deepctl" && chmod +x "$3/bin/deepctl"', + ' [ "$HANG_AT" = venv ] && hang; exit 0;;', + ' "-m pip") case "$*" in', + ' *" -U pip"*) [ "$HANG_AT" = pip-upgrade ] && hang;;', + ' *) [ "$HANG_AT" = pip-install ] && hang;;', + ' esac; exit 0;;', + 'esac', + 'exit 0', + '', + ].join('\n'); + for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) fakeExe(bin, name, fakePython); + const cache = path.join(home, 'c'); + const env = { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: cache, DEEPCTL_NPM_RUNNER: 'venv', HANG_AT: hangAt }; + // Where the launcher itself will put venvs and locks: XDG_CACHE_HOME + // on Linux, but ~/Library/Caches on macOS, which ignores it. + const root = launcher.cacheRoots({ env, home })[0]; + return { home, root, pidfile, env }; + } + + function start(env) { + const { spawn } = require('node:child_process'); + const child = spawn(process.execPath, [BIN], { env, stdio: ['ignore', 'pipe', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (d) => (stderr += d)); + child.stdout.resume(); + const exited = new Promise((r) => child.on('exit', (code, signal) => r({ code, signal }))); + return { child, exited, stderr: () => stderr }; + } + + for (const step of ['venv', 'pip-upgrade', 'pip-install']) { + test(`SIGTERM while the ${step} step runs: no orphan, no lock, no partial venv, dies by SIGTERM`, async () => { + const ws = workspace(step); + const run = start(ws.env); + await waitFor(() => fs.existsSync(ws.pidfile), `the ${step} step to start`); + const pids = fs.readFileSync(ws.pidfile, 'utf8').trim().split(/\s+/).map(Number); + assert.equal(pids.length, 2); + // The lock names the running step's process group (its shell's pid). + // It is recorded just after spawn(), which a busy machine can let the + // step outrun, so wait for it rather than assume it. + await waitForGroup(ws.root, pids[0]); + process.kill(run.child.pid, 'SIGTERM'); + const { code, signal } = await run.exited; + // Re-raised after cleanup, so the parent sees the signal itself. + assert.equal(signal, 'SIGTERM', run.stderr()); + assert.equal(code, null); + assert.match(run.stderr(), /setup interrupted \(SIGTERM\)/); + await waitFor(() => pids.every(gone), 'the setup step and its child to die', 3000); + assert.deepEqual(fs.readdirSync(ws.root), [], 'no lock and no partial venv left'); + }); + } + + test('SIGTERM while waiting for another setup\'s lock: exits at once, foreign lock untouched', async () => { + const ws = workspace('none'); + fs.mkdirSync(ws.root, { recursive: true, mode: 0o700 }); + const lock = path.join(ws.root, `${version}.lock`); + liveForeignLock(lock); + const owner = fs.readFileSync(path.join(lock, 'owner.json'), 'utf8'); + const run = start(ws.env); + await waitFor(() => /Waiting for another dg setup/.test(run.stderr()), 'the lock wait'); + const sentAt = Date.now(); + process.kill(run.child.pid, 'SIGTERM'); + const { code, signal } = await run.exited; + assert.equal(signal, 'SIGTERM', run.stderr()); + assert.equal(code, null); + assert.ok(Date.now() - sentAt < 2000, 'the wait is not a blocking sleep'); + assert.deepEqual(fs.readdirSync(ws.root), [`${version}.lock`]); + assert.equal(fs.readFileSync(path.join(lock, 'owner.json'), 'utf8'), owner); + }); + + const linuxProc = { skip: !fs.existsSync('/proc/self/stat') && 'needs /proc' }; + + test('after a SIGKILLed launcher, the next run stops the orphaned step, then builds', linuxProc, async () => { + const ws = workspace('pip-install'); + const first = start(ws.env); + await waitFor(() => fs.existsSync(ws.pidfile), 'the pip-install step to start'); + const pids = fs.readFileSync(ws.pidfile, 'utf8').trim().split(/\s+/).map(Number); + await waitForGroup(ws.root, pids[0]); + // SIGKILL cannot be handled: the step is orphaned with the lock held. + process.kill(first.child.pid, 'SIGKILL'); + await first.exited; + assert.ok(pids.every((pid) => !gone(pid)), 'the orphaned step is still running'); + const second = start({ ...ws.env, HANG_AT: 'none' }); + const { code } = await second.exited; + assert.equal(code, 0, second.stderr()); + await waitFor(() => pids.every(gone), 'the orphaned step to be killed', 3000); + assert.deepEqual(fs.readdirSync(ws.root).sort(), [version]); + }); + + test('a venv directory matches itself and paths inside it, not a prefix sibling', () => { + const dir = path.join('/c', 'venvs', '0.4.1'); + assert.equal(launcher.mentionsDir(dir, dir), true); + assert.equal(launcher.mentionsDir(path.join(dir, 'bin', 'python'), dir), true); + assert.equal(launcher.mentionsDir(path.join('/c', 'venvs', '0.4.10'), dir), false); + assert.equal(launcher.mentionsDir(path.join('/c', 'venvs', '0.4.10', 'bin', 'python'), dir), false); + assert.equal(launcher.mentionsDir(`${dir}.lock`, dir), false); + }); + + test('a stale lock for 0.4.1 leaves a 0.4.10 setup group alone', linuxProc, async () => { + const { spawn } = require('node:child_process'); + const root = tmpdir(); + const other = spawn('/bin/sh', ['-c', 'sleep 300; :', path.join(root, '0.4.10', 'bin', 'python')], { detached: true, stdio: 'ignore' }); + try { + const owner = JSON.stringify({ pid: 999999, host: os.hostname(), time: 1, pgid: other.pid }); + assert.equal(launcher.killOrphanedSetup(owner, path.join(root, '0.4.1')), false); + assert.equal(gone(other.pid), false); + } finally { + try { + process.kill(-other.pid, 'SIGKILL'); + } catch { + // Already gone. + } + } + }); + + test('a stale lock\'s process group is left alone unless it is building that venv', linuxProc, async () => { + const { spawn } = require('node:child_process'); + const root = tmpdir(); + const venvDir = path.join(root, '1.0.0'); + const groups = ['elsewhere', venvDir].map((arg) => spawn('/bin/sh', ['-c', 'sleep 300; :', arg], { detached: true, stdio: 'ignore' })); + const owner = (pgid) => JSON.stringify({ pid: 999999, host: os.hostname(), time: 1, pgid }); + try { + // Unrelated command line (a reused pgid): not killed. + assert.equal(launcher.killOrphanedSetup(owner(groups[0].pid), venvDir), false); + // Another host's pgid means nothing here. + assert.equal(launcher.killOrphanedSetup(JSON.stringify({ pid: 1, host: 'elsewhere', time: 1, pgid: groups[1].pid }), venvDir), false); + assert.equal(gone(groups[0].pid) || gone(groups[1].pid), false); + // Mentions the venv directory: the whole group goes. + assert.equal(launcher.killOrphanedSetup(owner(groups[1].pid), venvDir), true); + await waitFor(() => gone(groups[1].pid), 'the setup group to die', 3000); + assert.equal(gone(groups[0].pid), false); + } finally { + for (const g of groups) { + try { + process.kill(-g.pid, 'SIGKILL'); + } catch { + // Already gone. + } + } + } + }); +}); From 3a5ea7d8ab0e5007df6879c315c6d09db7b4eb22 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 30 Oct 2026 16:25:13 -0400 Subject: [PATCH 03/15] ci(release): publish @deepgram/cli to npm and gate releases on version bumps release-please now bumps npm/package.json with the root version. After PyPI serves that exact version, a publish-npm job publishes the package through trusted publishing. CI runs the launcher's tests on Node 20.0.0, 20, 22 and 24, and a cross-OS smoke test runs on Node 22. check_dependency_floors.py gains SIBLING_FLOORS: update and skills pin deepctl-core, and plugin pins deepctl-cmd-update, to the workspace version, and --fix raises them. Rule 2 now evaluates whole specifiers. Floors only hold if every changed package is actually released. So check_release_bumps.py fails when a package's src/ or non-workspace dependencies changed since its last release tag without a version bump. On the release PR it reports as the release-bump-gate check: sync-release-pr posts it as a commit status on the bot's commits, and a test.yml job covers commits people push. It runs again in the build job before PyPI. verify-published installs from PyPI alone, then runs deepctl and imports the launcher. --- .github/release-please-config.json | 7 +- .github/workflows/release.yml | 255 ++++++++++++++++++++++++++- .github/workflows/test.yml | 140 ++++++++++++++- CONTRIBUTING.md | 18 +- Makefile | 4 +- scripts/check_dependency_floors.py | 160 ++++++++++++++--- scripts/check_release_bumps.py | 221 +++++++++++++++++++++++ tests/unit/test_dependency_floors.py | 225 ++++++++++++++++++++++- tests/unit/test_release_bumps.py | 244 +++++++++++++++++++++++++ 9 files changed, 1238 insertions(+), 36 deletions(-) create mode 100644 scripts/check_release_bumps.py create mode 100644 tests/unit/test_release_bumps.py diff --git a/.github/release-please-config.json b/.github/release-please-config.json index fd5be2c1..553d9c3f 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -11,7 +11,12 @@ "include-component-in-tag": false, "make-latest": true, "extra-files": [ - "src/deepctl/__init__.py" + "src/deepctl/__init__.py", + { + "type": "json", + "path": "npm/package.json", + "jsonpath": "$.version" + } ] }, "packages/deepctl-core": { diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c44f6691..eb727ced 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,10 +40,17 @@ jobs: concurrency: group: sync-release-pr cancel-in-progress: false + # contents: push the sync commit. statuses: report release-bump-gate on + # the release PR's head (see the last step). + permissions: + contents: write + statuses: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: ref: ${{ fromJSON(needs.release-please.outputs.pr).headBranchName }} + # Every release tag, for the bump check below. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -65,7 +72,7 @@ jobs: # them; for an untouched release PR, re-run checks from the UI. If # releases ever need to go out unattended, switch this push to a # dedicated PAT or GitHub App token. - - name: Regenerate uv.lock and pin root floors + - name: Regenerate uv.lock and pin root and sibling floors run: | set -euo pipefail uv run python scripts/check_dependency_floors.py --fix @@ -81,10 +88,40 @@ jobs: fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add uv.lock pyproject.toml - git commit -m "chore: sync uv.lock and root dependency floors with release-please bumps" + git add uv.lock pyproject.toml packages/*/pyproject.toml + git commit -m "chore: sync uv.lock and dependency floors with release-please bumps" git push + # Is every package whose code or dependencies changed being bumped? + # This job runs on push to main, so failing it only reddens main; what + # can block the merge is a status on the release PR's own head commit. + # release-please and the push above both use GITHUB_TOKEN, whose + # commits trigger no pull_request workflows, so post that status here, + # on whatever HEAD this job left the branch at. Human pushes to the + # release PR do trigger workflows, and test.yml's release-bump-gate job + # reports the same context name for those commits. It only blocks the + # merge once `release-bump-gate` is a required check on main. + # Runs after the push so the lockfile sync above lands either way. + - name: Check every changed package is being released + run: | + set -uo pipefail + if uv run python scripts/check_release_bumps.py; then + STATE=success + DESCRIPTION="Every changed package is being released" + else + STATE=failure + DESCRIPTION="A package changed without a version bump; see the run log" + fi + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/$(git rev-parse HEAD)" \ + -f state="${STATE}" \ + -f context=release-bump-gate \ + -f description="${DESCRIPTION}" \ + -f target_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + || { echo "::error::Could not post the release-bump-gate status"; exit 1; } + [ "${STATE}" = success ] + env: + GH_TOKEN: ${{ github.token }} + build: name: Build packages needs: release-please @@ -92,6 +129,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + # Every release tag, for the bump check below. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -99,7 +139,27 @@ jobs: python-version: "3.11" - name: Install build dependencies - run: pip install build twine + run: pip install build twine packaging + + # The sync job above pins root and sibling floors on the release PR, + # but its push uses GITHUB_TOKEN and does not retrigger the PR's + # checks, so nothing proves the merged tree still satisfies them. + # Check again here: a stale floor would let pip pair a new package + # with an older published sibling that lacks the API it imports. + - name: Check dependency floors before publishing + run: python scripts/check_dependency_floors.py + + # Same check as the release PR's release-bump-gate status, again here + # as a last line of defence: that status only blocks the merge if it + # is a required check, and this run stops the PyPI upload regardless. + # Floors are compared with workspace versions, so they only hold if + # every package whose code changed is actually being released. A + # `chore:` commit under a package moves its code without a bump, and + # PyPI keeps serving the old code under the same version: a clean + # install then pairs new siblings with an old deepctl-core that has no + # deepctl_core.launcher. Refuse to publish in that state. + - name: Check every changed package is being released + run: python scripts/check_release_bumps.py - name: Build all packages run: make build @@ -204,12 +264,31 @@ jobs: # build job smoke-tests `deepctl --version`, but against the local # dist/ artifacts -- this is the only check against what PyPI serves, # and it is the last gate before three jobs advertise the release. - - name: Install for real and run the CLI + # + # A fresh venv with nothing but PyPI (no --find-links, no cache), so + # the sub-packages are the ones PyPI serves, not the ones just built. + # A retry covers a wheel the CDN has not finished propagating. + - name: Install for real from PyPI and run the CLI run: | set -euo pipefail VERSION="${TAG_NAME#v}" - /tmp/verify/bin/pip install --quiet --no-cache-dir "deepctl==${VERSION}" - /tmp/verify/bin/deepctl --version + for i in $(seq 1 5); do + rm -rf /tmp/real + python -m venv /tmp/real + if /tmp/real/bin/pip install --quiet --no-cache-dir "deepctl==${VERSION}"; then + break + fi + if [ "${i}" = 5 ]; then + echo "::error::pip install deepctl==${VERSION} from PyPI failed 5 times" + exit 1 + fi + echo "Install attempt ${i}/5 failed; retrying in 30s..." + sleep 30 + done + /tmp/real/bin/deepctl --version + # What the npm launcher and `dg update` rely on: a sibling released + # with stale code fails here with ModuleNotFoundError. + /tmp/real/bin/python -c "import deepctl_core.launcher, deepctl_cmd_update.installation" env: TAG_NAME: ${{ needs.release-please.outputs.tag_name }} @@ -345,3 +424,165 @@ jobs: --head "${BRANCH}" \ --title "Bump deepctl to ${VERSION}" \ --body-file "${BODY_FILE}" + + publish-npm: + name: Publish @deepgram/cli to npm + needs: [release-please, verify-published] + # The npm package is a launcher that runs `deepctl==` + # from PyPI, so it must never go out before that exact version installs. + # verify-published is that gate. Root tags only, same as mark-latest. + # + # Recovery: if this job fails (npm outage, trusted publisher not set up + # yet), fix the cause and use "Re-run failed jobs" on this workflow run. + # Job outputs from release-please are preserved across re-runs, and every + # step here is safe to repeat: an already-published version is skipped. + if: | + needs.release-please.outputs.release_created == 'true' && + startsWith(needs.release-please.outputs.tag_name, 'v') + runs-on: ubuntu-latest + # Trusted publishing: npm exchanges this job's OIDC token for a one-time + # publish credential, so no long-lived npm token exists to leak. The + # trusted publisher on npmjs.com is bound to this workflow file and this + # environment; renaming either breaks publishing until it is updated. + environment: npm-publish + permissions: + id-token: write + contents: read + timeout-minutes: 30 + env: + TAG_NAME: ${{ needs.release-please.outputs.tag_name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + + # No registry-url: it writes an .npmrc expecting NODE_AUTH_TOKEN, and an + # empty token there makes npm skip the OIDC exchange. + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + # Trusted publishing needs npm 11.5.1+; Node 22 bundles npm 10. Pinned + # exactly so a new npm release cannot change how a release publishes. + - name: Install an npm that supports trusted publishing + run: npm install -g npm@11.21.0 + + # release-please bumps npm/package.json through extra-files. If that + # ever drifts, publishing would ship a launcher pinned to a deepctl + # version that verify-published never checked. + - name: Check the package version matches the tag + id: version + run: | + set -euo pipefail + VERSION="${TAG_NAME#v}" + PKG_VERSION="$(node -p 'require("./npm/package.json").version')" + if [ "${PKG_VERSION}" != "${VERSION}" ]; then + echo "::error::npm/package.json is ${PKG_VERSION} but the tag is ${TAG_NAME}" + exit 1 + fi + if ! printf '%s' "${VERSION}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::${VERSION} is not a plain X.Y.Z version" + exit 1 + fi + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + + # npm refuses to republish a version, so a re-run after a later step + # failed would otherwise fail here forever. + - name: Check whether this version is already on npm + id: existing + run: | + set -euo pipefail + if [ "$(npm view "@deepgram/cli@${VERSION}" version 2>/dev/null || true)" = "${VERSION}" ]; then + echo "@deepgram/cli@${VERSION} is already published; skipping publish" + echo "published=true" >> "$GITHUB_OUTPUT" + else + echo "published=false" >> "$GITHUB_OUTPUT" + fi + env: + VERSION: ${{ steps.version.outputs.version }} + + # `npm publish` moves `latest` by default. A patch for an older line + # (0.4.x after 0.5.0 shipped) must not drag every new install back, so + # it goes out under a maintenance tag instead. Fail closed: only "the + # package has no latest yet" (E404) may fall through to latest. + - name: Choose the dist-tag + id: tag + run: | + set -euo pipefail + ERR="$(mktemp)" + if LATEST="$(npm view @deepgram/cli dist-tags.latest 2>"${ERR}")"; then + : + elif grep -q E404 "${ERR}"; then + LATEST="" + else + cat "${ERR}" + echo "::error::could not read the current latest dist-tag; refusing to guess" + exit 1 + fi + TAG=latest + if [ -n "${LATEST}" ]; then + ORDER="$(node -e ' + const parse = (v) => { + const [core, pre] = v.split("+")[0].split("-"); + return { n: core.split(".").map(Number), pre: pre !== undefined }; + }; + const a = parse(process.argv[1]); + const b = parse(process.argv[2]); + let c = 0; + for (let i = 0; i < 3 && c === 0; i++) c = Math.sign(a.n[i] - b.n[i]); + if (c === 0 && a.pre !== b.pre) c = a.pre ? -1 : 1; + console.log(c < 0 ? "lower" : "not-lower"); + ' "${VERSION}" "${LATEST}")" + if [ "${ORDER}" = "lower" ]; then + TAG="v$(printf '%s' "${VERSION}" | cut -d. -f1-2)-maint" + fi + fi + echo "latest is ${LATEST:-unset}; publishing ${VERSION} with dist-tag ${TAG}" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + env: + VERSION: ${{ steps.version.outputs.version }} + + - name: Publish + if: steps.existing.outputs.published != 'true' + working-directory: npm + run: npm publish --access public --provenance --tag "${DIST_TAG}" + env: + DIST_TAG: ${{ steps.tag.outputs.tag }} + + - name: Wait until the registry serves the version + run: | + set -euo pipefail + for i in $(seq 1 40); do + if [ "$(npm view "@deepgram/cli@${VERSION}" version 2>/dev/null || true)" = "${VERSION}" ]; then + echo "@deepgram/cli@${VERSION} is visible" + exit 0 + fi + echo "Waiting for @deepgram/cli@${VERSION} (${i}/40)..." + sleep 15 + done + echo "::error::@deepgram/cli@${VERSION} was not visible on the registry after 10 minutes" + exit 1 + env: + VERSION: ${{ steps.version.outputs.version }} + + # The end-to-end check a user would do: npx resolves the published + # tarball, the launcher hands off to uv, uv installs deepctl from PyPI. + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Run the published package + run: | + set -euo pipefail + for i in $(seq 1 10); do + if OUT="$(npx -y "@deepgram/cli@${VERSION}" --version 2>/dev/null)" && printf '%s' "${OUT}" | grep -qF "${VERSION}"; then + echo "${OUT}" + exit 0 + fi + echo "npx @deepgram/cli@${VERSION} not runnable yet (${i}/10)..." + sleep 30 + done + echo "Final attempt, with output:" + npx -y "@deepgram/cli@${VERSION}" --version || true + echo "::error::npx @deepgram/cli@${VERSION} --version did not report ${VERSION}" + exit 1 + env: + VERSION: ${{ steps.version.outputs.version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8dc2d477..2e978112 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -112,6 +112,142 @@ jobs: - name: Verify packages run: make verify-packages + # The launcher's unit tests on every Node version package.json's engines + # field admits. Linux only: the cross-OS job below covers platforms. + launcher-node: + name: npm launcher unit tests / Node ${{ matrix.node }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 20.0.0 is the engines floor itself. + node: ["20.0.0", "20", "22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + + # Unquoted: Node 20 does not expand --test globs itself, bash does. + - name: Unit tests + run: node --test npm/test/*.test.js + + launcher: + name: npm launcher / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + # A glob, not the directory: Node 22+ treats --test arguments as file + # patterns, and Node expands it itself where the shell does not. + - name: Unit tests + run: node --test "npm/test/*.test.js" + + # npm/LICENSE is what npmjs.com and the tarball show; it must not drift + # from the repository's. + - name: npm/LICENSE matches LICENSE + run: cmp LICENSE npm/LICENSE + + # Regression: the launcher once made the stdin it hands to deepctl + # non-blocking, so a reader that had to wait (an MCP host, `ffmpeg | + # dg listen`) failed. A stand-in deepctl checks fds 0-2 and reads input + # that arrives a second late. + - name: stdin stays blocking for deepctl + if: runner.os != 'Windows' + run: | + set -euo pipefail + FAKE="$(mktemp -d)" + cat > "${FAKE}/deepctl" <<'PY' + #!/usr/bin/env python3 + import fcntl, os, sys + flags = [bool(fcntl.fcntl(fd, fcntl.F_GETFL) & os.O_NONBLOCK) for fd in (0, 1, 2)] + print("nonblock=%s read=%r" % (flags, sys.stdin.read())) + PY + chmod +x "${FAKE}/deepctl" + OUT="$( (sleep 1; echo hello) | PATH="${FAKE}:${PATH}" DEEPCTL_NPM_RUNNER=path node npm/bin/dg.js)" + echo "${OUT}" + [ "${OUT}" = "nonblock=[False, False, False] read='hello\n'" ] + + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + # npm/package.json is already bumped on a release PR, before PyPI has + # that version, so run the launcher against the newest deepctl that + # PyPI actually serves. This checks the launcher, not the release. + - name: Smoke test against the latest deepctl on PyPI + run: | + set -euo pipefail + LATEST="$(node -e 'fetch("https://pypi.org/pypi/deepctl/json").then(r => r.json()).then(d => console.log(d.info.version))')" + echo "deepctl on PyPI: ${LATEST}" + echo "latest=${LATEST}" >> "$GITHUB_ENV" + OUT="$(DEEPCTL_NPM_PYPI_SPEC="deepctl==${LATEST}" node npm/bin/dg.js --version)" + echo "${OUT}" + printf '%s' "${OUT}" | grep -qF "${LATEST}" + + # The fallback for machines without uv: find Python, build a private + # venv, run from it. Windows exercises Scripts\deepctl.exe and the py + # launcher here. + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Smoke test the venv strategy + run: | + set -euo pipefail + # tr: Python on Windows ends lines with CRLF. + OUT="$(DEEPCTL_NPM_RUNNER=venv DEEPCTL_NPM_PYPI_SPEC="deepctl==${latest}" node npm/bin/dg.js --version | tr -d '\r')" + printf 'stdout: %s\n' "${OUT}" + # Exactly the version line: setup chatter must stay on stderr. + [ "${OUT}" = "deepctl, version ${latest}" ] + # Windows never reuses a venv: the one-run venv must be gone. + if [ "${RUNNER_OS}" = "Windows" ]; then + LEFT="$(ls -A "$(cygpath "${LOCALAPPDATA}")/deepgram-cli/venvs")" + [ -z "${LEFT}" ] || { echo "::error::one-run venv left behind: ${LEFT}"; exit 1; } + fi + + # The release PR's bump gate (scripts/check_release_bumps.py), for commits + # people push to the release PR, such as a CHANGELOG edit. Commits from + # release-please and the release workflow's sync job use GITHUB_TOKEN and + # trigger no workflows; release.yml's sync-release-pr posts the same + # `release-bump-gate` context as a commit status on those. Skipped on + # every other PR, which a required check treats as passing. Deliberately + # not in all-checks: on non-release PRs it never runs. + release-bump-gate: + name: release-bump-gate + if: github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-please--') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + with: + # The PR head, not the merge commit, and every release tag. + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Check every changed package is being released + run: | + pip install packaging + python scripts/check_release_bumps.py + # Single rollup so branch protection needs exactly one required check. # Requiring the matrix contexts directly means editing repo settings every # time a Python version or OS is added or dropped, and a required context @@ -123,7 +259,7 @@ jobs: # cancelled and skipped prerequisites are failures here too. all-checks: name: All checks - needs: [test, lint, floors, build-test] + needs: [test, lint, floors, build-test, launcher-node, launcher] if: always() runs-on: ubuntu-latest steps: @@ -133,6 +269,8 @@ jobs: echo "lint: ${{ needs.lint.result }}" echo "floors: ${{ needs.floors.result }}" echo "build-test: ${{ needs.build-test.result }}" + echo "launcher-node: ${{ needs.launcher-node.result }}" + echo "launcher: ${{ needs.launcher.result }}" - name: Fail unless every prerequisite succeeded if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a6f453d..2c2e2239 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,13 +70,29 @@ only installs what root depends on, so a published package missing from that list never reaches anyone. `make floors-check` fails on both that omission and a floor left below the workspace version; run `make floors-fix` to pin floors. +When a package starts importing a sibling API that only the sibling's newest +release has, add the pair to `SIBLING_FLOORS` in +`scripts/check_dependency_floors.py`. That floor then tracks the sibling's +workspace version the same way root floors do, and the release workflow checks +it again before building for PyPI. + +Floors only hold if every package whose code changed is released. +`scripts/check_release_bumps.py` fails when a package's `src/` or dependencies +changed since its last release tag but its version did not move (a `chore:` or +`refactor:` commit does not bump it). On the release PR it reports as the +`release-bump-gate` check: `sync-release-pr` posts it as a commit status on the +bot's commits, and `test.yml` runs it for commits people push. It blocks the +merge only while `release-bump-gate` is a required check on `main`. The build +job runs it again before anything reaches PyPI. Land a `fix:` commit +touching that package, or a `Release-As:` footer, and release again. + ### Releasing Releases are driven by release-please. Two things are worth knowing before you run one: **Land your release-notes edits as a commit, not a PR description edit.** -`sync-release-pr` regenerates `uv.lock` and the root dependency floors on the +`sync-release-pr` regenerates `uv.lock` and the root and sibling dependency floors on the release branch automatically, but it pushes with `GITHUB_TOKEN`, which by design retriggers nothing — so the PR's checks still reflect the bot's first commit and stay red on content that is now correct. Editing `CHANGELOG.md` on diff --git a/Makefile b/Makefile index 49b13b83..045a68fc 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,10 @@ readmes: ## Generate sub-package READMEs from pyproject.toml metadata readmes-check: ## Check sub-package READMEs are up to date python3 scripts/generate_readmes.py --check -floors-check: ## Check intra-workspace dependency floors (root must pin workspace versions) +floors-check: ## Check intra-workspace dependency floors (root and SIBLING_FLOORS must pin workspace versions) uv run python scripts/check_dependency_floors.py -floors-fix: ## Pin root dependency floors to the current workspace versions +floors-fix: ## Pin root and SIBLING_FLOORS dependency floors to the current workspace versions uv run python scripts/check_dependency_floors.py --fix # =================================================================== diff --git a/scripts/check_dependency_floors.py b/scripts/check_dependency_floors.py index 61da1c04..58926708 100644 --- a/scripts/check_dependency_floors.py +++ b/scripts/check_dependency_floors.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Check (or fix) intra-workspace dependency floors. -Three rules, learned from the 0.3.0 release (PRs #100/#102): +Four rules, the first three learned from the 0.3.0 release (PRs #100/#102): 1. **Root floors equal workspace versions.** `dg update` runs `pip install --upgrade deepctl`, and pip's default `only-if-needed` @@ -27,13 +27,25 @@ deliberately not shipped as part of the CLI goes in NOT_SHIPPED, so that intent is stated in the diff rather than inferred from an omission. +4. **Coupled sibling floors equal workspace versions.** Some sub-packages + import a sibling API that only exists in the sibling's newest release + (deepctl-cmd-update needs `deepctl_core.launcher`; deepctl-cmd-plugin + needs `InstallMethod.NPM` from deepctl-cmd-update). Rule 2 only stops a + floor from going *above* the sibling, so a stale floor lets pip pair the + new package with an old published sibling that lacks the API, which is an + ImportError at startup. Pairs listed in SIBLING_FLOORS are pinned like + root: the floor must equal the sibling's workspace version, and --fix + raises it. The release workflow runs the check again before building for + PyPI, so a release cannot publish with a stale pair. + Versions are compared with `packaging.version.Version`, not as strings, so a pre-release anywhere in the workspace (0.4.0rc1, 1.0.0.dev1, 0.3.0.post1) behaves like any other version instead of turning `--fix` into a loop the following check can never satisfy. -Run with --fix to rewrite root floors in place (used by the release -workflow's sync job so rule 1 holds automatically on every release PR). +Run with --fix to rewrite root floors and SIBLING_FLOORS pairs in place +(used by the release workflow's sync job so rules 1 and 4 hold automatically +on every release PR). """ from __future__ import annotations @@ -57,6 +69,7 @@ ) sys.exit(1) +from packaging.requirements import InvalidRequirement, Requirement from packaging.version import InvalidVersion, Version REPO = Path(__file__).resolve().parent.parent @@ -69,6 +82,17 @@ "deepctl-plugin-example", # sample plugin, installed on demand } +# Sub-package -> siblings whose floor must equal the sibling's workspace +# version (rule 4). Add a pair when a package starts importing an API that +# only the sibling's newest release has; the floor then tracks every release. +SIBLING_FLOORS: dict[str, tuple[str, ...]] = { + # deepctl_core.launcher (npm launcher detection) + "deepctl-cmd-update": ("deepctl-core",), + "deepctl-cmd-skills": ("deepctl-core",), + # deepctl_cmd_update.installation.InstallMethod.NPM + "deepctl-cmd-plugin": ("deepctl-cmd-update",), +} + # A PEP 508 requirement, narrowed to intra-workspace packages: the name, an # optional extras group, then whatever the author wrote after it. Matching # only bare `name>=version` used to make `deepctl-cmd-keys[extra]>=0.0.1` or @@ -98,16 +122,36 @@ class Dep: raw: str -def workspace_versions() -> dict[str, str]: - """Map package name -> current workspace version, per the manifest.""" - versions: dict[str, str] = {} +def workspace_packages() -> dict[str, tuple[str, Path]]: + """Map package name -> (current workspace version, pyproject path).""" + packages: dict[str, tuple[str, Path]] = {} for path in json.loads(MANIFEST.read_text(encoding="utf-8")): pyproject = REPO / ( "pyproject.toml" if path == "." else f"{path}/pyproject.toml" ) project = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"] - versions[project["name"]] = project["version"] - return versions + packages[project["name"]] = (project["version"], pyproject) + return packages + + +def workspace_versions() -> dict[str, str]: + """Map package name -> current workspace version, per the manifest.""" + return {name: version for name, (version, _) in workspace_packages().items()} + + +def rewrite_floor(text: str, dep: Dep, version: str) -> str | None: + """Return `text` with `dep`'s floor set to `version`, or None if unfixable. + + Rewrites the version inside the spec we matched, so any upper bound, + extra, or environment marker survives. A whole-spec literal replace + silently no-ops on those, and an unfixable floor has to fall through to + the problem list -- reporting "OK" after failing to fix is worse than + not fixing. + """ + new_raw = dep.raw.replace(dep.spec, f">={version}", 1) + if new_raw == dep.raw or f'"{dep.raw}"' not in text: + return None + return text.replace(f'"{dep.raw}"', f'"{new_raw}"', 1) def floors(pyproject: Path) -> list[Dep]: @@ -145,16 +189,37 @@ def parse(version: str) -> Version | None: return None +def excludes_current(pkg: str, dep: Dep, current: str) -> str | None: + """A problem if `dep`'s full specifier rejects the sibling's workspace version.""" + try: + spec = Requirement(dep.raw).specifier + except InvalidRequirement as err: + return f"{pkg}: {dep.raw!r} is not a valid requirement ({err})" + version = parse(current) + if version is None: + return f"{pkg}: {dep.name} workspace version {current} is not a valid PEP 440 version" + if spec.contains(version, prereleases=True): + return None + return ( + f"{pkg}: {dep.raw!r} excludes workspace version {current}" + " (unsatisfiable when released together)" + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--fix", action="store_true", - help="rewrite root pyproject.toml floors to the workspace versions", + help=( + "rewrite root pyproject.toml floors and SIBLING_FLOORS pairs to" + " the workspace versions" + ), ) args = parser.parse_args() - versions = workspace_versions() + packages = workspace_packages() + versions = {name: version for name, (version, _) in packages.items()} problems: list[str] = [] # Rule 1: root floors == workspace versions. @@ -188,14 +253,9 @@ def main() -> int: if floor_version == current_version: continue if args.fix: - # Rewrite the version inside the spec we matched, so any upper - # bound, extra, or environment marker survives. A whole-spec - # literal replace silently no-ops on those, and an unfixable - # floor has to fall through to `problems` -- reporting "OK" - # after failing to fix is worse than not fixing. - new_raw = dep.raw.replace(dep.spec, f">={current}", 1) - if new_raw != dep.raw and f'"{dep.raw}"' in fixed: - fixed = fixed.replace(f'"{dep.raw}"', f'"{new_raw}"', 1) + rewritten = rewrite_floor(fixed, dep, current) + if rewritten is not None: + fixed = rewritten continue problems.append( f"root floor {dep.name}>={dep.floor} != workspace version {current}" @@ -219,6 +279,11 @@ def main() -> int: ) continue if dep.floor is None: + # `==`, `~=`, `<` and friends: not a floor, but they can + # still exclude the sibling being released alongside. + problem = excludes_current(pkg_dir.name, dep, current) + if problem: + problems.append(problem) continue floor_version, current_version = parse(dep.floor), parse(current) if floor_version is None or current_version is None: @@ -231,6 +296,60 @@ def main() -> int: f"{pkg_dir.name}: floor {dep.name}>={dep.floor} exceeds" f" workspace version {current} (unsatisfiable)" ) + else: + # The rest of the spec (`,!=0.4.0`, `,<0.4`) can still exclude it. + problem = excludes_current(pkg_dir.name, dep, current) + if problem: + problems.append(problem) + + # Rule 4: coupled sibling floors == the sibling's workspace version. + for pkg_name, siblings in sorted(SIBLING_FLOORS.items()): + if pkg_name not in packages: + # Only reachable in a trimmed workspace (the tests build those); + # tests/unit/test_dependency_floors.py checks every name here + # against the real manifest so a rename cannot disable the rule. + continue + pyproject = packages[pkg_name][1] + pkg_text = pyproject.read_text(encoding="utf-8") + pkg_fixed = pkg_text + declared = {d.name: d for d in floors(pyproject)} + for sibling in siblings: + current = versions.get(sibling) + sibling_dep = declared.get(sibling) + if current is None: + problems.append( + f"SIBLING_FLOORS pairs {pkg_name} with {sibling}," + " which is not in the manifest" + ) + continue + if sibling_dep is None: + problems.append( + f"{pkg_name} must declare {sibling}>={current}" + " (it imports an API only the newest release has)" + ) + continue + if sibling_dep.floor is None: + problems.append( + f"{pkg_name}: {sibling_dep.raw!r} must be a `>=` floor so it can" + f" track {sibling}'s workspace version (write" + f" {sibling}>={current})" + ) + continue + if parse(sibling_dep.floor) == parse(current): + continue + if args.fix: + rewritten = rewrite_floor(pkg_fixed, sibling_dep, current) + if rewritten is not None: + pkg_fixed = rewritten + continue + problems.append( + f"{pkg_name}: floor {sibling}>={sibling_dep.floor} != workspace version" + f" {current} (pip could pair it with an older {sibling} that" + " lacks the API it imports)" + ) + if args.fix and pkg_fixed != pkg_text: + pyproject.write_text(pkg_fixed, encoding="utf-8") + print(f"fixed: sibling floors pinned to workspace versions in {pyproject}") # Rule 3: every published package is a root dependency. listed = {dep.name for dep in root_floors} @@ -246,8 +365,9 @@ def main() -> int: for problem in problems: print(f" - {problem}", file=sys.stderr) print( - "\nRun `make floors-fix` to pin stale root floors. Sub-package" - " floors and missing root dependencies are hand-maintained.", + "\nRun `make floors-fix` to pin stale root and SIBLING_FLOORS" + " floors. Other sub-package floors and missing dependencies are" + " hand-maintained.", file=sys.stderr, ) return 1 diff --git a/scripts/check_release_bumps.py b/scripts/check_release_bumps.py new file mode 100644 index 00000000..6962a553 --- /dev/null +++ b/scripts/check_release_bumps.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Fail if a package's code changed since its last release without a version bump. + +The dependency floors (scripts/check_dependency_floors.py) compare against +*workspace* versions, so they are only right when release-please bumps +every package whose code changed. It does not always: a `chore:` or +`refactor:` commit under a package moves its code without a release. PyPI +then keeps serving the old code under the same version number, and a +sibling that needs the new code installs next to it and fails at import. +On this branch, a clean `pip install` of the root wheel resolved +deepctl-core 0.3.0 from PyPI, which has no `deepctl_core.launcher`. + +So before anything is published: for every package in the release-please +manifest whose current version already has a release tag, `src/` and the +package's dependencies must be unchanged since that tag. A version with no +tag yet is about to be released from this tree, which is fine. + +Dependencies are compared parsed, not as text, so the release PR's own +version edit does not count. Floors on other workspace packages +(`deepctl-*>=X`) are left out of the comparison: the release sync job raises +them mechanically in packages it is not bumping, and +check_dependency_floors.py already governs them. Adding, removing or +re-pinning any other dependency without a bump is caught. + +Tag names follow .github/release-please-config.json: `v` for a +package with `include-component-in-tag: false` (the root), otherwise +`-v`. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - only taken on Python 3.10 + try: + import tomli as tomllib + except ModuleNotFoundError: + print( + "Python 3.11+ required (tomllib), or install tomli: pip install tomli", + file=sys.stderr, + ) + sys.exit(1) + +from packaging.requirements import Requirement + +REPO = Path(__file__).resolve().parent.parent +CONFIG = REPO / ".github" / "release-please-config.json" +MANIFEST = REPO / ".github" / ".release-please-manifest.json" + + +@dataclass(frozen=True) +class Package: + path: str # repo-relative, "." for the root + version: str + tag: str + + @property + def src(self) -> str: + return "src" if self.path == "." else f"{self.path}/src" + + @property + def pyproject(self) -> str: + return "pyproject.toml" if self.path == "." else f"{self.path}/pyproject.toml" + + +def dependency_set(pyproject_text: str) -> list[str]: + """A package's dependencies, normalized for comparison across releases. + + Covers `dependencies` and every `optional-dependencies` group. A + workspace package (`deepctl-*`) keeps its name, extras and marker but + loses its version specifier; see the module docstring for why. + """ + project = tomllib.loads(pyproject_text).get("project", {}) + groups = {"": project.get("dependencies", [])} + groups.update(project.get("optional-dependencies", {})) + out = [] + for group, deps in groups.items(): + for dep in deps: + req = Requirement(dep) + if req.name.lower().startswith("deepctl"): + req.specifier = type(req.specifier)() + out.append(f"{group}:{req}") + return sorted(out) + + +def tag_name(config: dict[str, Any], path: str, version: str) -> str: + """The tag release-please gives `path` at `version`.""" + entry = config.get("packages", {}).get(path, {}) + v = "v" if config.get("include-v-in-tag", True) else "" + if not entry.get("include-component-in-tag", True): + return f"{v}{version}" + return f"{entry.get('component', Path(path).name)}-{v}{version}" + + +def packages(repo: Path = REPO) -> list[Package]: + config = json.loads((repo / CONFIG.relative_to(REPO)).read_text(encoding="utf-8")) + manifest = json.loads( + (repo / MANIFEST.relative_to(REPO)).read_text(encoding="utf-8") + ) + out = [] + for path in manifest: + pyproject = repo / ( + "pyproject.toml" if path == "." else f"{path}/pyproject.toml" + ) + version = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"][ + "version" + ] + out.append(Package(path, version, tag_name(config, path, version))) + return out + + +def unbumped( + pkgs: list[Package], + tag_exists: Callable[[str], bool], + changed_since: Callable[[str, str], bool], + deps_changed: Callable[[str, Package], bool] = lambda _tag, _pkg: False, +) -> list[str]: + """Problems for packages whose src/ or dependencies moved past their tag.""" + problems = [] + for pkg in pkgs: + if not tag_exists(pkg.tag): + continue # This version has not been released yet. + if changed_since(pkg.tag, pkg.src): + problems.append( + f"{pkg.path}: {pkg.src}/ changed since {pkg.tag}, but the version is" + f" still {pkg.version}, so PyPI would keep serving the old code" + ) + if deps_changed(pkg.tag, pkg): + problems.append( + f"{pkg.path}: dependencies in {pkg.pyproject} changed since {pkg.tag}," + f" but the version is still {pkg.version}, so PyPI would keep" + " serving the old dependency list" + ) + return problems + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True + ) + + +def git_tag_exists(repo: Path) -> Callable[[str], bool]: + """A tag lookup that fails loudly rather than reading an error as "no tag". + + "No tag" means "first release, pass", so a git that cannot answer (not a + repository, a shallow checkout without tags) must not look like one. + """ + if not _git(repo, "tag", "--list").stdout.strip(): + raise RuntimeError( + f"no tags in {repo}: fetch them (actions/checkout fetch-depth: 0)" + ) + + def exists(tag: str) -> bool: + result = _git(repo, "rev-parse", "-q", "--verify", f"refs/tags/{tag}") + if result.returncode not in (0, 1): + raise RuntimeError(f"git rev-parse {tag} failed: {result.stderr.strip()}") + return result.returncode == 0 + + return exists + + +def git_changed_since(repo: Path) -> Callable[[str, str], bool]: + def changed(tag: str, path: str) -> bool: + result = _git(repo, "diff", "--quiet", tag, "HEAD", "--", path) + if result.returncode not in (0, 1): + raise RuntimeError(f"git diff {tag} failed: {result.stderr.strip()}") + return result.returncode == 1 + + return changed + + +def git_deps_changed(repo: Path) -> Callable[[str, Package], bool]: + def changed(tag: str, pkg: Package) -> bool: + old = _git(repo, "show", f"{tag}:{pkg.pyproject}") + new = _git(repo, "show", f"HEAD:{pkg.pyproject}") + for result, ref in ((old, tag), (new, "HEAD")): + if result.returncode != 0: + raise RuntimeError( + f"git show {ref}:{pkg.pyproject} failed: {result.stderr.strip()}" + ) + return dependency_set(old.stdout) != dependency_set(new.stdout) + + return changed + + +def main(repo: Path = REPO) -> int: + problems = unbumped( + packages(repo), + git_tag_exists(repo), + git_changed_since(repo), + git_deps_changed(repo), + ) + if problems: + print("release bump check FAILED:", file=sys.stderr) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + print( + "\nLand a `fix:` commit touching each package (or a `Release-As:`" + " footer) so release-please bumps it, then release again.", + file=sys.stderr, + ) + return 1 + print( + "release bumps OK: no package ships changed code or dependencies under an old version" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else REPO)) diff --git a/tests/unit/test_dependency_floors.py b/tests/unit/test_dependency_floors.py index cb1cbae5..6ade2457 100644 --- a/tests/unit/test_dependency_floors.py +++ b/tests/unit/test_dependency_floors.py @@ -38,10 +38,7 @@ def make_workspace( deps = ", ".join(f'"{dep}"' for dep in root_deps) (tmp_path / "pyproject.toml").write_text( - "[project]\n" - 'name = "deepctl"\n' - 'version = "1.0.0"\n' - f"dependencies = [{deps}]\n", + f'[project]\nname = "deepctl"\nversion = "1.0.0"\ndependencies = [{deps}]\n', encoding="utf-8", ) @@ -240,3 +237,223 @@ def test_final_release_floor_above_a_pre_release_is_unsatisfiable( result = run_guard(workspace) assert result.returncode == 1 assert "exceeds workspace version 0.4.0rc1 (unsatisfiable)" in result.stderr + + +# --------------------------------------------------------------------------- +# Rule 4: coupled sibling floors (SIBLING_FLOORS) track the sibling's +# workspace version. deepctl-cmd-update imports deepctl_core.launcher, which +# no published deepctl-core had; with a stale floor pip can pair the new +# update package with an old core and `dg` fails at import. +# --------------------------------------------------------------------------- + +COUPLED = { + "deepctl-core": "0.4.0", + "deepctl-cmd-update": "0.3.0", + "deepctl-cmd-skills": "0.1.0", + "deepctl-cmd-plugin": "0.2.0", +} + + +def write_package(workspace: Path, name: str, deps: list[str]) -> Path: + pyproject = workspace / "packages" / name / "pyproject.toml" + quoted = ", ".join(f'"{dep}"' for dep in deps) + pyproject.write_text( + f'[project]\nname = "{name}"\nversion = "{COUPLED[name]}"\n' + f"dependencies = [{quoted}]\n", + encoding="utf-8", + ) + return pyproject + + +def coupled_workspace(tmp_path: Path, update: str, skills: str, plugin: str) -> Path: + workspace = make_workspace( + tmp_path, + root_deps=[f"{name}>={version}" for name, version in COUPLED.items()], + packages=COUPLED, + ) + write_package(workspace, "deepctl-cmd-update", [update]) + write_package(workspace, "deepctl-cmd-skills", [skills]) + write_package(workspace, "deepctl-cmd-plugin", ["deepctl-core>=0.1.0", plugin]) + return workspace + + +def package_dependencies(workspace: Path, name: str) -> list[str]: + text = (workspace / "packages" / name / "pyproject.toml").read_text( + encoding="utf-8" + ) + match = re.search(r"^dependencies = (\[.*\])$", text, re.MULTILINE) + assert match, text + deps: list[str] = ast.literal_eval(match.group(1)) + return deps + + +def test_stale_sibling_floor_fails(tmp_path: Path) -> None: + """The 0.3.0-era floor that let pip pick a core without launcher.py.""" + workspace = coupled_workspace( + tmp_path, + update="deepctl-core>=0.1.10", + skills="deepctl-core>=0.4.0", + plugin="deepctl-cmd-update>=0.3.0", + ) + + result = run_guard(workspace) + assert result.returncode == 1 + assert ( + "deepctl-cmd-update: floor deepctl-core>=0.1.10 != workspace version 0.4.0" + in result.stderr + ) + + +def test_missing_sibling_dependency_fails(tmp_path: Path) -> None: + """plugin imports deepctl_cmd_update, so it has to declare it.""" + workspace = coupled_workspace( + tmp_path, + update="deepctl-core>=0.4.0", + skills="deepctl-core>=0.4.0", + plugin="packaging>=21.0", + ) + + result = run_guard(workspace) + assert result.returncode == 1 + assert "deepctl-cmd-plugin must declare deepctl-cmd-update>=0.3.0" in result.stderr + + +def test_fix_raises_sibling_floors_then_check_passes(tmp_path: Path) -> None: + """--fix rewrites every SIBLING_FLOORS pair and leaves other deps alone.""" + workspace = coupled_workspace( + tmp_path, + update="deepctl-core>=0.1.10", + skills="deepctl-core[extra]>=0.2.19,<1", + plugin="deepctl-cmd-update>=0.2.7", + ) + + fix = run_guard(workspace, "--fix") + assert fix.returncode == 0, fix.stderr + assert package_dependencies(workspace, "deepctl-cmd-update") == [ + "deepctl-core>=0.4.0" + ] + assert package_dependencies(workspace, "deepctl-cmd-skills") == [ + "deepctl-core[extra]>=0.4.0,<1" + ] + # Not a SIBLING_FLOORS pair: still hand-maintained, untouched. + assert package_dependencies(workspace, "deepctl-cmd-plugin") == [ + "deepctl-core>=0.1.0", + "deepctl-cmd-update>=0.3.0", + ] + + check = run_guard(workspace) + assert check.returncode == 0, check.stderr + + +def test_fix_does_not_invent_a_missing_sibling_dependency(tmp_path: Path) -> None: + """Adding a dependency is a code change; --fix only moves floors.""" + workspace = coupled_workspace( + tmp_path, + update="deepctl-core>=0.4.0", + skills="deepctl-core>=0.4.0", + plugin="packaging>=21.0", + ) + + fix = run_guard(workspace, "--fix") + assert fix.returncode == 1 + assert "deepctl-cmd-plugin must declare deepctl-cmd-update" in fix.stderr + + +def test_sibling_floor_names_exist_in_the_real_manifest() -> None: + """A rename must not quietly turn rule 4 off. + + The guard skips SIBLING_FLOORS entries whose package is absent so the + trimmed workspaces above stay valid; this pins the real names instead. + """ + import importlib.util + + spec = importlib.util.spec_from_file_location("floors_guard", SCRIPT) + assert spec and spec.loader + guard = importlib.util.module_from_spec(spec) + # @dataclass resolves annotations through sys.modules[cls.__module__]. + sys.modules[spec.name] = guard + try: + spec.loader.exec_module(guard) + finally: + del sys.modules[spec.name] + + names = set(guard.workspace_versions()) + for pkg_name, siblings in guard.SIBLING_FLOORS.items(): + assert pkg_name in names, pkg_name + for sibling in siblings: + assert sibling in names, sibling + + +def test_real_workspace_passes_the_guard() -> None: + """`make floors-check` on the repo itself, as CI and the release run it.""" + result = subprocess.run( + [sys.executable, str(SCRIPT)], capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "spec", + [ + "deepctl-core==0.3.0", + "deepctl-core~=0.3.0", + "deepctl-core<0.4", + "deepctl-core>=0.1,!=0.4.0", + ], +) +def test_non_floor_sub_package_specifier_that_excludes_the_sibling_fails( + tmp_path: Path, spec: str +) -> None: + """Rule 2 used to skip anything that was not a plain `>=`.""" + workspace = make_workspace( + tmp_path, + root_deps=["deepctl-core>=0.4.0", "deepctl-cmd-keys>=0.1.0"], + packages={"deepctl-core": "0.4.0", "deepctl-cmd-keys": "0.1.0"}, + ) + keys = workspace / "packages" / "deepctl-cmd-keys" / "pyproject.toml" + keys.write_text( + '[project]\nname = "deepctl-cmd-keys"\nversion = "0.1.0"\n' + f'dependencies = ["{spec}"]\n', + encoding="utf-8", + ) + + result = run_guard(workspace) + assert result.returncode == 1 + assert f"{spec!r} excludes workspace version 0.4.0" in result.stderr + + +@pytest.mark.parametrize( + "spec", ["deepctl-core~=0.4.0", "deepctl-core<1", "deepctl-core==0.4.*"] +) +def test_non_floor_sub_package_specifier_that_admits_the_sibling_passes( + tmp_path: Path, spec: str +) -> None: + workspace = make_workspace( + tmp_path, + root_deps=["deepctl-core>=0.4.0", "deepctl-cmd-keys>=0.1.0"], + packages={"deepctl-core": "0.4.0", "deepctl-cmd-keys": "0.1.0"}, + ) + keys = workspace / "packages" / "deepctl-cmd-keys" / "pyproject.toml" + keys.write_text( + '[project]\nname = "deepctl-cmd-keys"\nversion = "0.1.0"\n' + f'dependencies = ["{spec}"]\n', + encoding="utf-8", + ) + + result = run_guard(workspace) + assert result.returncode == 0, result.stderr + + +def test_pinned_sibling_floor_is_named_for_what_it_is(tmp_path: Path) -> None: + """A `==` pin on a SIBLING_FLOORS pair is not a missing dependency.""" + workspace = coupled_workspace( + tmp_path, + update="deepctl-core==0.4.0", + skills="deepctl-core>=0.4.0", + plugin="deepctl-cmd-update>=0.3.0", + ) + + result = run_guard(workspace) + assert result.returncode == 1 + assert "'deepctl-core==0.4.0' must be a `>=` floor" in result.stderr + assert "must declare" not in result.stderr diff --git a/tests/unit/test_release_bumps.py b/tests/unit/test_release_bumps.py new file mode 100644 index 00000000..bffe8985 --- /dev/null +++ b/tests/unit/test_release_bumps.py @@ -0,0 +1,244 @@ +"""Tests for scripts/check_release_bumps.py, the pre-publish bump gate.""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / "scripts" / "check_release_bumps.py" + + +def load_gate(): + spec = importlib.util.spec_from_file_location("release_bumps", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # @dataclass needs the module registered + try: + spec.loader.exec_module(module) + finally: + del sys.modules[spec.name] + return module + + +gate = load_gate() + +CONFIG = { + "include-v-in-tag": True, + "packages": { + ".": {"component": "deepctl", "include-component-in-tag": False}, + "packages/deepctl-core": { + "component": "deepctl-core", + "include-component-in-tag": True, + }, + }, +} + + +def test_tag_names_follow_the_release_please_config() -> None: + assert gate.tag_name(CONFIG, ".", "0.4.0") == "v0.4.0" + assert ( + gate.tag_name(CONFIG, "packages/deepctl-core", "0.3.0") == "deepctl-core-v0.3.0" + ) + # release-please's default includes the component. + assert ( + gate.tag_name({"packages": {}}, "packages/deepctl-x", "1.0.0") + == "deepctl-x-v1.0.0" + ) + + +def test_the_real_config_names_every_manifest_package() -> None: + names = {pkg.path: pkg.tag for pkg in gate.packages(REPO)} + assert names["."].startswith("v") + assert names["packages/deepctl-core"].startswith("deepctl-core-v") + + +CORE = gate.Package("packages/deepctl-core", "0.3.0", "deepctl-core-v0.3.0") +ROOT = gate.Package(".", "0.4.0", "v0.4.0") + + +def test_changed_code_under_a_released_version_fails() -> None: + problems = gate.unbumped([CORE], lambda tag: True, lambda tag, path: True) + assert len(problems) == 1 + assert "packages/deepctl-core/src/ changed since deepctl-core-v0.3.0" in problems[0] + + +def test_unchanged_code_passes() -> None: + assert gate.unbumped([CORE, ROOT], lambda tag: True, lambda tag, path: False) == [] + + +def test_a_version_without_a_tag_is_a_first_release_and_passes() -> None: + def changed(tag: str, path: str) -> bool: + raise AssertionError("no diff without a tag") + + assert gate.unbumped([CORE], lambda tag: False, changed) == [] + + +def test_only_src_is_compared() -> None: + seen = [] + gate.unbumped( + [CORE, ROOT], lambda tag: True, lambda tag, path: seen.append(path) or False + ) + assert seen == ["packages/deepctl-core/src", "src"] + + +def pyproject_text(version: str, deps: list[str], optional: dict | None = None) -> str: + text = f'[project]\nname = "deepctl-core"\nversion = "{version}"\ndependencies = {json.dumps(deps)}\n' + for group, items in (optional or {}).items(): + text += f"[project.optional-dependencies]\n{group} = {json.dumps(items)}\n" + return text + + +def test_a_version_bump_alone_is_not_a_dependency_change() -> None: + old = pyproject_text("0.3.0", ["click>=8", "rich>=13"]) + new = pyproject_text("0.3.1", ["rich>=13", "click>=8"]) + assert gate.dependency_set(old) == gate.dependency_set(new) + + +def test_workspace_floors_are_ignored_but_their_presence_is_not() -> None: + old = pyproject_text("0.3.0", ["deepctl-core>=0.2.19"]) + # The release sync raising a sibling floor is mechanical: not a change. + assert gate.dependency_set(old) == gate.dependency_set( + pyproject_text("0.3.0", ["deepctl-core>=0.4.0"]) + ) + # Adding or dropping a workspace dependency is. + assert gate.dependency_set(old) != gate.dependency_set( + pyproject_text("0.3.0", ["deepctl-core>=0.2.19", "deepctl-cmd-update>=0.2.7"]) + ) + + +@pytest.mark.parametrize( + "new_deps", + [["click>=8", "httpx>=0.27"], ["click>=9"], ["click>=8; python_version<'3.12'"]], +) +def test_third_party_changes_are_dependency_changes(new_deps: list[str]) -> None: + old = pyproject_text("0.3.0", ["click>=8"]) + assert gate.dependency_set(old) != gate.dependency_set( + pyproject_text("0.3.0", new_deps) + ) + + +def test_optional_dependencies_count() -> None: + old = pyproject_text("0.3.0", [], {"audio": ["numpy>=1"]}) + assert gate.dependency_set(old) != gate.dependency_set( + pyproject_text("0.3.0", [], {"audio": ["numpy>=2"]}) + ) + + +def test_a_dependency_change_under_a_released_version_fails() -> None: + problems = gate.unbumped( + [CORE], lambda tag: True, lambda tag, path: False, lambda tag, pkg: True + ) + assert problems == [ + "packages/deepctl-core: dependencies in packages/deepctl-core/pyproject.toml changed" + " since deepctl-core-v0.3.0, but the version is still 0.3.0, so PyPI would keep" + " serving the old dependency list" + ] + + +@pytest.mark.skipif(shutil.which("git") is None, reason="needs git") +def test_against_a_real_git_repository(tmp_path: Path) -> None: + """The CLI end to end: tags, a bump, and a forgotten bump.""" + + def git(*args: str) -> None: + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=t", + "-c", + "user.email=t@t", + *args, + ], + check=True, + capture_output=True, + ) + + def write(rel: str, text: str) -> None: + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def pyproject(rel: str, name: str, version: str) -> None: + write(rel, f'[project]\nname = "{name}"\nversion = "{version}"\n') + + def run() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), str(tmp_path)], capture_output=True, text=True + ) + + write(".github/release-please-config.json", json.dumps(CONFIG)) + write( + ".github/.release-please-manifest.json", + json.dumps({".": "0.4.0", "packages/deepctl-core": "0.3.0"}), + ) + pyproject("pyproject.toml", "deepctl", "0.4.0") + pyproject("packages/deepctl-core/pyproject.toml", "deepctl-core", "0.3.0") + write("src/deepctl/__init__.py", "") + write("packages/deepctl-core/src/deepctl_core/__init__.py", "") + git("init", "-q") + git("add", "-A") + git("commit", "-qm", "release") + git("tag", "v0.4.0") + git("tag", "deepctl-core-v0.3.0") + + # Fresh tags at HEAD: nothing changed. + assert run().returncode == 0 + + # Docs-only and test-only changes are fine. + write("packages/deepctl-core/README.md", "docs\n") + git("add", "-A") + git("commit", "-qm", "docs") + assert run().returncode == 0 + + # A new third-party dependency under the released version. + write( + "packages/deepctl-core/pyproject.toml", + '[project]\nname = "deepctl-core"\nversion = "0.3.0"\ndependencies = ["httpx>=0.27"]\n', + ) + git("add", "-A") + git("commit", "-qm", "chore: add httpx") + result = run() + assert result.returncode == 1 + assert ( + "dependencies in packages/deepctl-core/pyproject.toml changed" in result.stderr + ) + + # Code changed under the released version too: the 0.3.0 trap. + write("packages/deepctl-core/src/deepctl_core/launcher.py", "X = 1\n") + git("add", "-A") + git("commit", "-qm", "chore: add launcher") + result = run() + assert result.returncode == 1 + assert ( + "packages/deepctl-core/src/ changed since deepctl-core-v0.3.0" in result.stderr + ) + assert "src/ changed since v0.4.0" not in result.stderr + + # Bumped (no tag for 0.3.1 yet): about to be released, so it passes. + pyproject("packages/deepctl-core/pyproject.toml", "deepctl-core", "0.3.1") + git("add", "-A") + git("commit", "-qm", "chore: release core 0.3.1") + assert run().returncode == 0, run().stderr + + +def test_a_repository_without_tags_fails_instead_of_passing(tmp_path: Path) -> None: + """A shallow checkout has no tags; that must not read as "first release".""" + if shutil.which("git") is None: + pytest.skip("needs git") + subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True) + with pytest.raises(RuntimeError, match="no tags"): + gate.git_tag_exists(tmp_path) + + +def test_not_a_repository_fails(tmp_path: Path) -> None: + with pytest.raises(RuntimeError): + gate.git_tag_exists(tmp_path)("v1.0.0") From e90e1dd6b64c3464e3d43265c43adb3b133ff0e3 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 30 Oct 2026 16:25:13 -0400 Subject: [PATCH 04/15] docs: document the npm install The README and llms-full.txt document `npm install -g @deepgram/cli` and `npx @deepgram/cli`. llms-full.txt still said 0.2.4, so it moves to web/src/data/ and a prerendered endpoint now builds it with the version from pyproject.toml, the same way index.astro already reads it. The build fails if that version can't be read. The doc-command test follows the file to its new path. --- README.md | 6 ++++++ tests/unit/test_command_examples.py | 2 +- web/{public => src/data}/llms-full.txt | 8 +++++--- web/src/pages/index.astro | 8 +------- web/src/pages/llms-full.txt.ts | 21 +++++++++++++++++++++ web/src/utils/cliVersion.ts | 13 +++++++++++++ 6 files changed, 47 insertions(+), 11 deletions(-) rename web/{public => src/data}/llms-full.txt (97%) create mode 100644 web/src/pages/llms-full.txt.ts create mode 100644 web/src/utils/cliVersion.ts diff --git a/README.md b/README.md index 46a2f16d..74fb5153 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,19 @@ iwr https://deepgram.com/install.ps1 -useb | iex pip install deepctl # pip uv tool install deepctl # uv pipx install deepctl # pipx +npm install -g @deepgram/cli # npm (installs dg and deepgram) ``` +The npm package is a launcher for the same Python CLI. It needs +[uv](https://docs.astral.sh/uv/) or Python 3.10+ with `venv`, and sets the CLI +up on first run. See [npm/README.md](npm/README.md). + ### Try Without Installing ```bash uvx deepctl --help pipx run deepctl --help +npx @deepgram/cli --help ``` ## Getting Started diff --git a/tests/unit/test_command_examples.py b/tests/unit/test_command_examples.py index 45de904f..4c959b50 100644 --- a/tests/unit/test_command_examples.py +++ b/tests/unit/test_command_examples.py @@ -207,7 +207,7 @@ def test_example_parses( DOC_FILES = [ "README.md", "web/public/llms.txt", - "web/public/llms-full.txt", + "web/src/data/llms-full.txt", ] # Templates, not runnable commands: `dg ... -o json | jq`, `dg diff --git a/web/public/llms-full.txt b/web/src/data/llms-full.txt similarity index 97% rename from web/public/llms-full.txt rename to web/src/data/llms-full.txt index 5b76e4dc..b0019ad2 100644 --- a/web/public/llms-full.txt +++ b/web/src/data/llms-full.txt @@ -1,7 +1,7 @@ # Deepgram CLI — Full Reference > The official command-line interface for Deepgram's speech AI platform. -> Version: 0.2.4 | License: MIT | Python 3.10+ required +> Version: {{CLI_VERSION}} | License: MIT | Python 3.10+ required > GitHub: https://github.com/deepgram/cli --- @@ -23,10 +23,12 @@ iwr deepgram.com/install.ps1 | iex pip install deepctl # pip pipx install deepctl # pipx (isolated) uvx deepctl --help # uvx (run without install) -brew install deepgram/tap/deepctl # Homebrew +brew install deepgram/tap/deepgram # Homebrew +npm install -g @deepgram/cli # npm (needs uv or Python 3.10+; installs dg, deepgram) +npx @deepgram/cli --help # npx (run without install) ``` -Aliases: `dg`, `deepctl`, `deepgram` +Aliases: `dg`, `deepctl`, `deepgram` (the npm package installs only `dg` and `deepgram`) --- diff --git a/web/src/pages/index.astro b/web/src/pages/index.astro index cacd44a1..082f8896 100644 --- a/web/src/pages/index.astro +++ b/web/src/pages/index.astro @@ -1,12 +1,6 @@ --- import '../styles/global.css'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; - -// Read version from root pyproject.toml at build time — always in sync, -// no manual updates needed. process.cwd() is web/ during `npm run build`. -const _toml = readFileSync(join(process.cwd(), '..', 'pyproject.toml'), 'utf-8'); -const cliVersion = _toml.match(/^version = "([^"]+)"/m)?.[1] ?? '0.0.0'; +import { cliVersion } from '../utils/cliVersion'; // ── Command showcase cards ──────────────────────────────────────────────── const ICONS = { diff --git a/web/src/pages/llms-full.txt.ts b/web/src/pages/llms-full.txt.ts new file mode 100644 index 00000000..1ef6ae9c --- /dev/null +++ b/web/src/pages/llms-full.txt.ts @@ -0,0 +1,21 @@ +import type { APIRoute } from 'astro'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { cliVersion } from '../utils/cliVersion'; + +export const prerender = true; + +// llms-full.txt is authored in src/data/ and served from here so its +// version line comes from pyproject.toml instead of going stale every release. +const TOKEN = '{{CLI_VERSION}}'; + +export const GET: APIRoute = () => { + const template = readFileSync(join(process.cwd(), 'src', 'data', 'llms-full.txt'), 'utf-8'); + if (!template.includes(TOKEN)) { + throw new Error(`src/data/llms-full.txt is missing the ${TOKEN} placeholder`); + } + + return new Response(template.replaceAll(TOKEN, cliVersion), { + headers: { 'Content-Type': 'text/plain; charset=utf-8' }, + }); +}; diff --git a/web/src/utils/cliVersion.ts b/web/src/utils/cliVersion.ts new file mode 100644 index 00000000..0b468c55 --- /dev/null +++ b/web/src/utils/cliVersion.ts @@ -0,0 +1,13 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +// Read the CLI version from the root pyproject.toml at build time, so pages +// never carry a hand-edited version. process.cwd() is web/ during `npm run build`. +const toml = readFileSync(join(process.cwd(), '..', 'pyproject.toml'), 'utf-8'); + +const match = toml.match(/^version = "([^"]+)"/m); +if (!match) { + throw new Error('Could not read the CLI version from the root pyproject.toml'); +} + +export const cliVersion = match[1]; From c9d61407b82d6f13d90ae158e6c8ece9b505687a Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:12:51 -0400 Subject: [PATCH 05/15] fix(npm): print launcher errors as ERROR and HINT lines Launcher failures printed as `dg: ` with the hint on a bare second line. They now print as `ERROR: ` and `HINT: `, the same two-line form the CLI uses, so a script or a person can tell the problem from the fix. That covers setup failures, an interrupted setup, a deepctl that can't start, and a deepctl that crashed. --- npm/lib/launcher.js | 10 +++++----- npm/test/launcher.test.js | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 11e63256..214318e2 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -1143,15 +1143,15 @@ async function run(argv = process.argv.slice(2), opts = {}) { // The setup child is gone, its partial venv removed and the lock // released (buildVenv's finally); now die from the same signal. const sig = session.interrupted; - stderr.write(`dg: setup interrupted (${sig}).\n`); + stderr.write(`ERROR: setup interrupted (${sig}).\n`); if (isWindows(process.platform)) process.exit(signalExitCode(sig)); dieBySignal(sig, signalExitCode(sig)); return; } // Filesystem surprises (a cache directory changing underneath us) get // the same one-line treatment as expected failures, not a stack trace. - const hint = err instanceof LauncherError && err.hint ? `\n${err.hint}` : ''; - stderr.write(`dg: ${String(err.message || err).trimStart()}${hint}\n`); + const hint = err instanceof LauncherError && err.hint ? `\nHINT: ${err.hint}` : ''; + stderr.write(`ERROR: ${String(err.message || err).trimStart()}${hint}\n`); process.exit(1); } @@ -1164,7 +1164,7 @@ async function run(argv = process.argv.slice(2), opts = {}) { session.child = child; child.on('error', (err) => { - stderr.write(`dg: could not start ${runner.command}: ${err.message}\n`); + stderr.write(`ERROR: could not start ${runner.command}: ${err.message}\n`); cleanup(); process.exit(1); }); @@ -1175,7 +1175,7 @@ async function run(argv = process.argv.slice(2), opts = {}) { removeHandlers(); const action = exitAction(code, signal); // A crash otherwise surfaces only as a bare exit status (139 and so on). - if (CRASH_SIGNALS.has(signal)) stderr.write(`dg: ${runner.command} crashed (${signal}).\n`); + if (CRASH_SIGNALS.has(signal)) stderr.write(`ERROR: ${runner.command} crashed (${signal}).\n`); if (action.signal) { // Die the same way the child did. dieBySignal(action.signal, action.exit); diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index 9be2ab03..ff0fce02 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -918,7 +918,7 @@ describe('bin/dg.js end to end', posixOnly, () => { const r = runBin({ PATH: '/nonexistent', HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); assert.equal(r.status, 1); assert.equal(r.stdout, ''); - assert.match(r.stderr, /needs uv or Python 3\.10\+/); + assert.match(r.stderr, /^ERROR: The Deepgram CLI needs uv or Python 3\.10\+/); }); test('passes arguments, env and exit code through a fake uv', () => { @@ -1017,7 +1017,7 @@ describe('bin/dg.js end to end', posixOnly, () => { fakeExe(bin, 'uv', '#!/bin/sh\nkill -SEGV $$\n'); const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); assert.equal(r.signal, 'SIGSEGV'); - assert.match(r.stderr, /dg: .*uv crashed \(SIGSEGV\)\./); + assert.match(r.stderr, /^ERROR: .*uv crashed \(SIGSEGV\)\./m); }); test('a venv build step killed by a signal says so', { skip: process.platform === 'win32' && 'POSIX signals' }, () => { From 57cb1de9cbd0aa241327151d07be544fb78b87f9 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:12:59 -0400 Subject: [PATCH 06/15] ci: correct the checkout pin comments and pin uv in the npm launcher jobs The launcher and release jobs this branch adds pin actions/checkout to a commit whose tag is v7.0.1, but their comments said v4. They now name the tag the SHA points at, and the jobs that install uv pin its version (0.13.6) instead of taking the newest release. --- .github/workflows/release.yml | 4 +++- .github/workflows/test.yml | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb727ced..d24d5a03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -452,7 +452,7 @@ jobs: env: TAG_NAME: ${{ needs.release-please.outputs.tag_name }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # No registry-url: it writes an .npmrc expecting NODE_AUTH_TOKEN, and an # empty token there makes npm skip the OIDC exchange. @@ -568,6 +568,8 @@ jobs: # tarball, the launcher hands off to uv, uv installs deepctl from PyPI. - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.13.6" - name: Run the published package run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e978112..912eb8c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -123,7 +123,7 @@ jobs: # 20.0.0 is the engines floor itself. node: ["20.0.0", "20", "22", "24"] steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 @@ -145,7 +145,7 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 @@ -184,6 +184,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.13.6" # npm/package.json is already bumped on a release PR, before PyPI has # that version, so run the launcher against the newest deepctl that @@ -232,7 +234,7 @@ jobs: if: github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-please--') runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # The PR head, not the merge commit, and every release tag. ref: ${{ github.event.pull_request.head.sha }} From e6cfd817df101d081914f0fb2c72f823d2112384 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:13:07 -0400 Subject: [PATCH 07/15] fix(npm): install the CLI only from hash-locked requirements The launcher used to install deepctl== and let pip or uv resolve everything else from the index. Each release now ships requirements-.txt in the npm package: deepctl and its workspace packages with the hashes of the files PyPI serves for them (read from its JSON API after the release is verified there), and every other dependency pinned with uv.lock's hashes. The release's publish-npm job writes it with npm/scripts/write_requirements.py, cross-checks it against the files that run built, and checks that the tarball contains it. Setup installs only from that file, with --require-hashes, --no-deps and --only-binary :all:, so a download that differs from the release is refused, nothing unlisted is installed, and no build backend runs. `uv tool run --with-requirements` ignores hashes, so with uv the launcher builds the same cached venv with `uv venv` and `uv pip install`, which check them. Setup runs in a directory the launcher owns, uv skips project configuration discovery (--no-config, with the user's or the machine's uv.toml passed back explicitly) and Python runs with -I, so running dg inside a repository can't change what gets installed, while pip.conf, uv.toml and PIP_*/UV_* mirror settings still apply. The pip fallback no longer upgrades pip first, since that download would be unhashed. Python is limited to 3.10 to 3.14, the versions the lock has wheels for. A package without its requirements file installs nothing. DEEPCTL_NPM_PYPI_SPEC, which CI uses to test against another deepctl, keeps the unhashed path. CI gains a Windows and macOS job that runs the hash-locked install from a local index (npm/scripts/local_index.py). --- .github/workflows/release.yml | 45 ++++- .github/workflows/test.yml | 77 +++++++- .gitignore | 2 + npm/README.md | 60 ++++-- npm/lib/launcher.js | 171 +++++++++++++---- npm/package.json | 3 +- npm/scripts/local_index.py | 94 ++++++++++ npm/scripts/write_requirements.py | 219 ++++++++++++++++++++++ npm/test/launcher.test.js | 276 ++++++++++++++++++++++++---- tests/unit/test_npm_requirements.py | 152 +++++++++++++++ 10 files changed, 1006 insertions(+), 93 deletions(-) create mode 100644 npm/scripts/local_index.py create mode 100644 npm/scripts/write_requirements.py create mode 100644 tests/unit/test_npm_requirements.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d24d5a03..1fdeca8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -445,9 +445,11 @@ jobs: # trusted publisher on npmjs.com is bound to this workflow file and this # environment; renaming either breaks publishing until it is updated. environment: npm-publish + # actions: read only to look up when this run attempt started (below). permissions: id-token: write contents: read + actions: read timeout-minutes: 30 env: TAG_NAME: ${{ needs.release-please.outputs.tag_name }} @@ -485,6 +487,41 @@ jobs: fi echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + # This attempt's build, used only to cross-check what PyPI serves. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.13.6" + + # The launcher installs only what this file lists, checked against its + # hashes: deepctl's own packages with the hashes of the files PyPI + # serves (its JSON API), everything else from uv.lock. Not hashes of + # dist/: wheels aren't byte-reproducible, and a version PyPI already + # had (skip-existing, or an earlier attempt of this run) keeps its + # first files. Files PyPI received during this attempt must match + # dist/ exactly. Without the file in the tarball, the launcher + # refuses to install anything. + - name: Write the hash-locked requirements file + run: | + set -euo pipefail + STARTED="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" --jq .run_started_at)" + python3 npm/scripts/write_requirements.py --dist dist/ --uploaded-since "${STARTED}" + (cd npm && npm pack --dry-run --json) > pack.json + # shellcheck disable=SC2016 # ${...} below is a JavaScript template + node -e ' + const files = JSON.parse(require("fs").readFileSync("pack.json", "utf8"))[0].files.map((f) => f.path); + const want = `requirements-${process.env.VERSION}.txt`; + if (!files.includes(want)) { console.error(`::error::${want} is not in the npm tarball`); process.exit(1); } + ' + env: + VERSION: ${{ steps.version.outputs.version }} + GH_TOKEN: ${{ github.token }} + # npm refuses to republish a version, so a re-run after a later step # failed would otherwise fail here forever. - name: Check whether this version is already on npm @@ -565,12 +602,8 @@ jobs: VERSION: ${{ steps.version.outputs.version }} # The end-to-end check a user would do: npx resolves the published - # tarball, the launcher hands off to uv, uv installs deepctl from PyPI. - - name: Install uv - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 - with: - version: "0.13.6" - + # tarball, and the launcher has uv install deepctl from PyPI, checked + # against the tarball's requirements file. - name: Run the published package run: | set -euo pipefail diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 912eb8c9..fb9f3167 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -222,6 +222,80 @@ jobs: [ -z "${LEFT}" ] || { echo "::error::one-run venv left behind: ${LEFT}"; exit 1; } fi + # The release path, end to end, without PyPI: build this commit's + # packages, serve them and their locked dependencies from a local index + # (simple pages plus the JSON API write_requirements.py reads), write the + # hash-locked requirements file against it, and run the launcher through + # uv and through the pip fallback. The `launcher` job's smoke tests use + # the unhashed DEEPCTL_NPM_PYPI_SPEC override; this one does not. + launcher-hashed: + name: npm launcher hash-locked install / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + defaults: + run: + shell: bash + env: + DEEPCTL_TELEMETRY_DISABLED: "1" + DEEPCTL_TELEMETRY_DSN: "http://k@127.0.0.1:9/1" + UV_PYTHON_DOWNLOADS: never + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.13.6" + + # One step: the index server only lives as long as the step's shell. + - name: Install through the launcher from a local index + run: | + set -euo pipefail + uv build --all-packages -q -o dist + uv export --frozen --no-dev --no-emit-workspace --no-header --no-annotate > third.txt + # Wheels for every Python the launcher may pick (it prefers the + # newest supported one, and runners have several installed). + for v in 3.12 3.13 3.14; do + python -m pip download -q --disable-pip-version-check --only-binary :all: --no-deps --require-hashes --python-version "${v}" -r third.txt -d wheels + done + python npm/scripts/local_index.py --out idx dist wheels + python -m http.server -b 127.0.0.1 -d idx 8765 > /dev/null 2>&1 & + SERVER=$! + trap 'kill "${SERVER}"' EXIT + for _ in $(seq 1 30); do curl -sf http://127.0.0.1:8765/simple/ > /dev/null && break; sleep 1; done + python npm/scripts/write_requirements.py --index-url http://127.0.0.1:8765 --dist dist + VERSION="$(node -p 'require("./npm/package.json").version')" + export UV_INDEX_URL=http://127.0.0.1:8765/simple PIP_INDEX_URL=http://127.0.0.1:8765/simple + for runner in uvx venv; do + # A fresh HOME per runner, so the venv run cannot reuse the venv + # the uv run cached (macOS keeps it under ~/Library/Caches). + # tr: Python on Windows ends lines with CRLF. + mkdir -p "${RUNNER_TEMP}/home-${runner}" + OUT="$(HOME="${RUNNER_TEMP}/home-${runner}" DEEPCTL_NPM_RUNNER="${runner}" node npm/bin/dg.js --version | tr -d '\r')" + printf '%s: %s\n' "${runner}" "${OUT}" + [ "${OUT}" = "deepctl, version ${VERSION}" ] + done + # Windows never reuses a venv: the one-run venvs must be gone. + if [ "${RUNNER_OS}" = "Windows" ]; then + LEFT="$(ls -A "$(cygpath "${LOCALAPPDATA}")/deepgram-cli/venvs")" + [ -z "${LEFT}" ] || { echo "::error::one-run venv left behind: ${LEFT}"; exit 1; } + fi + # The release PR's bump gate (scripts/check_release_bumps.py), for commits # people push to the release PR, such as a CHANGELOG edit. Commits from # release-please and the release workflow's sync job use GITHUB_TOKEN and @@ -261,7 +335,7 @@ jobs: # cancelled and skipped prerequisites are failures here too. all-checks: name: All checks - needs: [test, lint, floors, build-test, launcher-node, launcher] + needs: [test, lint, floors, build-test, launcher-node, launcher, launcher-hashed] if: always() runs-on: ubuntu-latest steps: @@ -273,6 +347,7 @@ jobs: echo "build-test: ${{ needs.build-test.result }}" echo "launcher-node: ${{ needs.launcher-node.result }}" echo "launcher: ${{ needs.launcher.result }}" + echo "launcher-hashed: ${{ needs.launcher-hashed.result }}" - name: Fail unless every prerequisite succeeded if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} diff --git a/.gitignore b/.gitignore index 388075d8..fde13384 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ eggs/ lib/ # The npm launcher's sources, not a build output. !/npm/lib/ +# Written by the release (npm/scripts/write_requirements.py). +/npm/requirements-*.txt lib64/ parts/ sdist/ diff --git a/npm/README.md b/npm/README.md index 93b260c6..33a00baa 100644 --- a/npm/README.md +++ b/npm/README.md @@ -28,18 +28,38 @@ same version as the npm package: 1. If an earlier run already set up a private Python environment for this version, it runs that. This is fast and works offline. -2. Otherwise, if [uv](https://docs.astral.sh/uv/) is installed, it runs - `uv tool run --from deepctl== deepctl`. uv brings its own Python. -3. Otherwise it finds Python 3.10 or newer and builds a private venv under - your cache directory (`~/.cache/deepgram-cli` or - `~/Library/Caches/deepgram-cli`). The first run takes about a minute. +2. Otherwise, if [uv](https://docs.astral.sh/uv/) is installed, uv builds that + environment. uv brings its own Python, and setup takes a few seconds. +3. Otherwise it finds Python 3.10 to 3.14 and builds the environment with + `venv` and `pip`. The first run takes about a minute. + +Either way, the environment lives under your cache directory +(`~/.cache/deepgram-cli` or `~/Library/Caches/deepgram-cli`), and setup +installs exactly the packages listed in the npm package's +`requirements-.txt`, each checked against its sha256 hash, and only +as prebuilt wheels (`--require-hashes --no-deps --only-binary :all:`). A +download that doesn't match the release is refused and nothing runs. Setup +runs in a directory the launcher owns, so running `dg` inside a cloned +repository can't change what gets installed: uv ignores any `uv.toml` or +`pyproject.toml` there. + +Your own package-index settings still apply, so a company mirror keeps +working: + +- pip: your user and global `pip.conf` (`pip config set global.index-url …`), + `PIP_CONFIG_FILE` and `PIP_*` variables such as `PIP_INDEX_URL`. +- uv: `UV_*` variables such as `UV_INDEX_URL` or `UV_DEFAULT_INDEX`, + `UV_CONFIG_FILE`, and one `uv.toml`: your user file + (`~/.config/uv/uv.toml`, or `%APPDATA%\uv\uv.toml` on Windows) if it + exists, otherwise the system file (`/etc/uv/uv.toml`, or + `%PROGRAMDATA%\uv\uv.toml`). On Windows, step 1 is skipped: the launcher can't verify who controls a cache -directory there, so it never runs a venv left by an earlier run. Without uv, -every run builds a fresh venv under `%LOCALAPPDATA%\deepgram-cli` and deletes -it afterwards, which takes about a minute each time. Install uv to avoid that. -The launcher needs a real `uv.exe` or `python.exe` on `PATH`: pyenv-win's -`python.bat` shims and `uv.cmd` wrappers are skipped. +directory there, so it never runs an environment left by an earlier run. Every +run builds a fresh one under `%LOCALAPPDATA%\deepgram-cli` and deletes it +afterwards. With uv that takes a few seconds; without it, about a minute each +time, so install uv. The launcher needs a real `uv.exe` or `python.exe` on +`PATH`: pyenv-win's `python.bat` shims and `uv.cmd` wrappers are skipped. Launcher messages go to stderr, so stdout carries only the CLI's own output (`dg ... -o json`, `dg mcp`). @@ -51,7 +71,7 @@ One of: - [uv](https://docs.astral.sh/uv/getting-started/installation/) (recommended): `curl -LsSf https://astral.sh/uv/install.sh | sh`, or on Windows `powershell -c "irm https://astral.sh/uv/install.ps1 | iex"` -- Python 3.10 or newer with `venv`. On Debian and Ubuntu that means +- Python 3.10 to 3.14 with `venv`. On Debian and Ubuntu that means `sudo apt install python3-venv`. Node.js 20 or newer runs the launcher. @@ -72,10 +92,24 @@ Update the npm version with `npm install -g @deepgram/cli@latest`, or run | Variable | Effect | | --- | --- | | `DEEPCTL_NPM_RUNNER=uvx\|venv\|path` | Force one strategy. `path` runs a `deepctl` you installed yourself from your `PATH`. | -| `DEEPCTL_NPM_PYPI_SPEC` | Replace the requirement installed from PyPI (default `deepctl==`). Meant for testing. | +| `DEEPCTL_NPM_PYPI_SPEC` | Replace the requirement installed from PyPI (default: the pinned, hash-checked `requirements-.txt`). Meant for testing: an override isn't hash-checked. | | `XDG_CACHE_HOME` | Cache directory base on Linux. | -## Known limitation: nohup and ignored signals +## Known limitations + +### Platforms without prebuilt wheels + +Setup installs prebuilt wheels only, never source builds. A few of the CLI's +dependencies don't publish wheels for every platform, so setup fails there +with "no matching distribution" or "no wheels": + +- Intel Macs and Windows on ARM64: `cryptography` has no wheel for them. +- Python 3.15: not supported yet; the launcher picks Python 3.10 to 3.14. + +On those machines, install the CLI another way that can build those packages +from source, such as `pipx install deepctl` or `brew install deepgram/tap/deepgram`. + +### nohup and ignored signals `nohup dg ...`, or a script that runs `dg` in the background after `trap '' INT`, does not protect the CLI the way it protects a native program. diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 214318e2..67d2021d 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -14,6 +14,10 @@ const { setTimeout: delay } = require('node:timers/promises'); const PKG_DIR = path.resolve(__dirname, '..'); const MIN_PYTHON = [3, 10]; +// Exclusive: the release's locked dependencies have no wheels for 3.15 yet, +// and setup installs wheels only (--only-binary :all:). +const MAX_PYTHON = [3, 15]; +const PYTHON_RANGE = `>=${MIN_PYTHON.join('.')},<${MAX_PYTHON.join('.')}`; const LOCK_WAIT_MS = 5 * 60 * 1000; const LOCK_STALE_MS = 10 * 60 * 1000; // No first-run install takes this long; past it a same-host lock is @@ -196,6 +200,32 @@ function pypiSpec(env, version) { return env.DEEPCTL_NPM_PYPI_SPEC || `deepctl==${version}`; } +// Every release ships requirements-.txt next to package.json: the +// exact deepctl distributions and dependencies of that release, each with +// its sha256 hashes. Setup installs only from it, with --require-hashes and +// --no-deps, so a download that differs by a byte from what was released +// is refused and nothing outside the list is installed. Returns {file}, +// or null for a DEEPCTL_NPM_PYPI_SPEC override (CI runs against a deepctl +// other than the packaged one): there is nothing to hash that against. +function lockedRequirements(env, version, opts = {}) { + if (env.DEEPCTL_NPM_PYPI_SPEC) return null; + const file = opts.requirementsFile || path.join(opts.pkgDir || PKG_DIR, `requirements-${version}.txt`); + const isFile = opts.isRequirementsFile || ((f) => { + try { + return fs.statSync(f).isFile(); + } catch { + return false; + } + }); + if (!isFile(file)) { + throw new LauncherError( + `This @deepgram/cli package is missing ${path.basename(file)}, so dg can't install the CLI safely.`, + `Reinstall it: npm install -g @deepgram/cli@${version}`, + ); + } + return { file }; +} + // Venvs are keyed by version. An overridden spec gets its own directory so // a CI run against a different deepctl never poisons the real cache entry. function venvKey(env, version) { @@ -321,13 +351,13 @@ function pythonCandidates(opts = {}) { candidates.push({ command: f, args: [] }); continue; } - for (const minor of [13, 12, 11, 10]) candidates.push({ command: f, args: [`-3.${minor}`] }); - // Newer than 3.13, or a launcher configured with only a default. + for (const minor of [14, 13, 12, 11, 10]) candidates.push({ command: f, args: [`-3.${minor}`] }); + // A launcher configured with only a default. candidates.push({ command: f, args: ['-3'] }); } return candidates; } - for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) { + for (const name of ['python3.14', 'python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) { for (const exe of whichAll(name, opts)) { // Running Apple's stub pops the "install command line developer // tools" dialog instead of failing quietly. @@ -369,10 +399,13 @@ async function execProbe(command, args, session) { } async function probePython(candidate, exec = execProbe) { - const result = await exec(candidate.command, [...candidate.args, '-c', PROBE_CODE]); + // -I: the current directory (an arbitrary project) must not shadow the + // venv or ensurepip modules the probe imports. + const result = await exec(candidate.command, [...candidate.args, '-I', '-c', PROBE_CODE]); const version = parseProbeOutput(result.stdout); if (!version) return { version: null, ok: false }; - return { version, ok: result.status === 0 && versionAtLeast(version, MIN_PYTHON), hasVenv: result.status === 0 }; + const supported = versionAtLeast(version, MIN_PYTHON) && !versionAtLeast(version, MAX_PYTHON); + return { version, ok: result.status === 0 && supported, hasVenv: result.status === 0 }; } // Returns the first usable Python plus what was seen on the way, so the @@ -822,12 +855,12 @@ function releaseLock(lockDir, owner) { const CRASH_SIGNALS = new Set(['SIGSEGV', 'SIGBUS', 'SIGILL', 'SIGFPE', 'SIGABRT']); -async function runStep(command, args, what, hint, opts = {}) { +async function runStep(command, args, what, hint, opts = {}, cwd = undefined) { const session = opts.session; throwIfInterrupted(session); // stdout goes to stderr: `dg -o json` and `dg mcp` own stdout. A process // group of its own (POSIX) lets an interrupt take down pip's children too. - const options = { stdio: ['ignore', 2, 2], windowsHide: true, detached: !isWindows(process.platform) }; + const options = { stdio: ['ignore', 2, 2], windowsHide: true, detached: !isWindows(process.platform), cwd }; const result = await superviseChild(command, args, options, session, opts.spawn); throwIfInterrupted(session); if (result.error) throw new LauncherError(`${what} failed: ${result.error.message}.`, hint); @@ -838,7 +871,9 @@ async function runStep(command, args, what, hint, opts = {}) { } const VENV_HINT = 'On Debian or Ubuntu, install python3-venv (sudo apt install python3-venv), or install uv instead.'; +const UV_VENV_HINT = `uv could not create a Python ${PYTHON_RANGE} environment; see its message above. If no such Python is installed: uv python install 3.12`; const NETWORK_HINT = 'Check your network connection and that PyPI is reachable, then run dg again.'; +const INSTALL_HINT = `Check that PyPI (or your package index) is reachable, then run dg again. A hash mismatch above means a download did not match this release, so nothing was installed. "No matching distribution" or "no wheels" means this platform or Python has no prebuilt package for one of the CLI's dependencies (see "Known limitations" in the npm package README); use Python 3.10 to 3.14, or install the CLI another way.`; // Removes other versions' venvs, but only ones nobody has used for a long // time: a dg pinned to an older npm version (npx @deepgram/cli@0.4.0, a @@ -874,7 +909,7 @@ function pruneSiblings(root, keep, now = Date.now()) { // Builds a venv for this run only, in a directory created fresh with an // unpredictable name, so nothing another process prepared can be picked up. // The caller deletes it once deepctl exits. -async function buildPrivateVenv({ python, key, spec, version, opts = {} }) { +async function buildPrivateVenv({ installer, key, spec, locked, version, opts = {} }) { const stderr = opts.stderr || fdStderr; const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; const base = cacheRootSpecs(opts)[0].dir; @@ -887,9 +922,10 @@ async function buildPrivateVenv({ python, key, spec, version, opts = {} }) { throw new LauncherError(`No writable directory for the Deepgram CLI: ${err.message}`, 'Install uv instead: it needs no setup on each run.'); } const layout = venvLayout(dir, opts.platform); - stderr.write(`Setting up Deepgram CLI ${version} for this run (~1 min each time; install uv to skip it: ${UV_INSTALL_WINDOWS})...\n`); + if (installer.uv) stderr.write(`Setting up Deepgram CLI ${version} for this run...\n`); + else stderr.write(`Setting up Deepgram CLI ${version} for this run (~1 min each time; install uv to speed it up: ${UV_INSTALL_WINDOWS})...\n`); try { - await installInto(python, layout, spec, opts); + await installInto(installer, layout, spec, locked, opts, { cwd: dir }); } catch (err) { fs.rmSync(dir, { recursive: true, force: true }); throw err; @@ -925,18 +961,72 @@ function pruneOneRunVenvs(base, p = path, now = Date.now()) { } } -async function installInto(python, layout, spec, opts) { - const pip = [layout.python, '-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q']; - await runStep(python.command, [...python.args, '-m', 'venv', layout.dir], 'Creating the Python venv', VENV_HINT, opts); - await runStep(pip[0], [...pip.slice(1), '-U', 'pip'], 'Upgrading pip', NETWORK_HINT, opts); - await runStep(pip[0], [...pip.slice(1), spec], `Installing ${spec}`, NETWORK_HINT, opts); +// uv's own configuration files, without its project discovery: --no-config +// stops uv reading uv.toml or pyproject.toml from the directory it runs in +// and its parents (a cloned repository, or a /tmp anyone can write to), and +// this passes the user's or the machine's uv.toml back explicitly, so a +// mirror configured there still applies. UV_CONFIG_FILE, when set, is read +// by uv itself. +function uvConfigArgs(env, opts = {}) { + if (env.UV_CONFIG_FILE) return []; + const platform = opts.platform || process.platform; + const p = isWindows(platform) ? path.win32 : path.posix; + const exists = opts.configExists || fs.existsSync; + const home = opts.home || os.homedir(); + const candidates = []; + if (isWindows(platform)) { + if (env.APPDATA && p.isAbsolute(env.APPDATA)) candidates.push(p.join(env.APPDATA, 'uv', 'uv.toml')); + if (env.PROGRAMDATA && p.isAbsolute(env.PROGRAMDATA)) candidates.push(p.join(env.PROGRAMDATA, 'uv', 'uv.toml')); + } else { + const xdg = env.XDG_CONFIG_HOME && p.isAbsolute(env.XDG_CONFIG_HOME) ? env.XDG_CONFIG_HOME : p.join(home, '.config'); + candidates.push(p.join(xdg, 'uv', 'uv.toml')); + for (const dir of (env.XDG_CONFIG_DIRS || '/etc/xdg').split(':').filter((d) => p.isAbsolute(d))) candidates.push(p.join(dir, 'uv', 'uv.toml')); + candidates.push('/etc/uv/uv.toml'); + } + const found = candidates.find((f) => exists(f)); + return found ? ['--config-file', found] : []; +} + +// Creates the venv with uv or with Python's own venv module, then installs +// into it. With a requirements file (every release), pip or uv installs +// exactly what it lists and checks every download against its hashes; +// --no-deps keeps anything unlisted out. --only-binary :all: means no +// source build, whose build backend would be an unhashed download that +// runs code. The venv's bundled pip is used as is: upgrading it would be +// an unhashed download of its own. +// +// Nothing a project can ship applies: every step runs in `where.cwd` (a +// directory the launcher created, never the user's); uv ignores uv.toml and +// pyproject.toml found there (--no-config) but reads the user's or the +// machine's uv.toml (uvConfigArgs) and UV_* variables; Python runs with -I, +// so no module in the cwd can shadow venv or pip. pip has no per-project +// configuration: its user and global pip.conf, PIP_CONFIG_FILE and PIP_* +// variables apply as usual. +async function installInto(installer, layout, spec, locked, opts, where = {}) { + const env = opts.env || process.env; + const source = [...(locked ? ['--require-hashes', '--no-deps', '-r', locked.file] : [spec]), '--only-binary', ':all:']; + const what = `Installing ${spec}`; + const hint = locked ? INSTALL_HINT : NETWORK_HINT; + if (installer.uv) { + const cache = where.privateCache ? ['--cache-dir', where.privateCache] : []; + const config = uvConfigArgs(env, opts); + const uv = (sub) => [...sub, '--no-config', ...config, '--quiet', ...cache]; + await runStep(installer.uv, [...uv(['venv']), '--allow-existing', '--python', PYTHON_RANGE, layout.dir], 'Creating the Python venv', UV_VENV_HINT, opts, where.cwd); + await runStep(installer.uv, [...uv(['pip', 'install']), '--python', layout.python, ...source], what, hint, opts, where.cwd); + return; + } + const { python } = installer; + await runStep(python.command, [...python.args, '-I', '-m', 'venv', layout.dir], 'Creating the Python venv', VENV_HINT, opts, where.cwd); + const noCache = where.privateCache ? ['--no-cache-dir'] : []; + const pip = [layout.python, '-I', '-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...noCache, ...source]; + await runStep(pip[0], pip.slice(1), what, hint, opts, where.cwd); } // The lock is released in `finally`, not from process.on('exit'): that hook // never runs when the process dies from a signal. run() turns signals into // an Interrupted error instead, so this unwinds normally. -async function buildVenv({ python, key, spec, version, opts = {} }) { - if (!cacheReuse(opts)) return buildPrivateVenv({ python, key, spec, version, opts }); +async function buildVenv({ installer, key, spec, locked, version, opts = {} }) { + if (!cacheReuse(opts)) return buildPrivateVenv({ installer, key, spec, locked, version, opts }); const stderr = opts.stderr || fdStderr; const prepare = opts.prepareRoot || ((spec) => prepareRoot(spec, opts)); const rootSpec = cacheRootSpecs(opts).find(prepare); @@ -955,9 +1045,9 @@ async function buildVenv({ python, key, spec, version, opts = {} }) { if (venvUsable(layout)) return layout; // Half-built, or broken by a removed base interpreter: start over. fs.rmSync(layout.dir, { recursive: true, force: true }); - stderr.write(`Setting up Deepgram CLI ${version} (one-time, ~1 min)...\n`); + stderr.write(`Setting up Deepgram CLI ${version} (one-time${installer.uv ? '' : ', ~1 min'})...\n`); try { - await installInto(python, layout, spec, opts); + await installInto(installer, layout, spec, locked, opts, { cwd: root }); fs.writeFileSync(layout.marker, `${spec}\n`); } catch (err) { fs.rmSync(layout.dir, { recursive: true, force: true }); @@ -1001,26 +1091,33 @@ function exitAction(code, signal, platform = process.platform) { // --- Plan ----------------------------------------------------------------- function installHelp(seen) { - const lines = ['', 'The Deepgram CLI needs uv or Python 3.10+ (with venv) to run. Fix one of:', '']; + const tooNew = seen.find((s) => s.version && versionAtLeast(s.version, MAX_PYTHON)); + if (tooNew && !seen.some((s) => s.ok)) { + return new LauncherError( + `Python ${tooNew.version.join('.')} is too new for this release of the Deepgram CLI, and no Python 3.10 to 3.14 was found.`, + 'Install Python 3.12, or uv (which brings its own Python), then run dg again.', + ); + } + const lines = ['', 'The Deepgram CLI needs uv or Python 3.10 to 3.14 (with venv) to run. Fix one of:', '']; lines.push(' 1. Install uv (recommended), then run dg again:'); lines.push(' curl -LsSf https://astral.sh/uv/install.sh | sh'); lines.push(` ${UV_INSTALL_WINDOWS} (Windows)`); lines.push(' On Windows, the launcher only runs real uv.exe / python.exe files: pyenv-win\'s'); lines.push(' python.bat shims and uv.cmd are skipped, since a .bat or .cmd needs a shell'); lines.push(' that would re-parse your arguments.'); - const noVenv = seen.find((s) => s.version && versionAtLeast(s.version, MIN_PYTHON) && !s.hasVenv); + const noVenv = seen.find((s) => s.version && versionAtLeast(s.version, MIN_PYTHON) && !versionAtLeast(s.version, MAX_PYTHON) && !s.hasVenv); if (noVenv) { lines.push(` 2. Python ${noVenv.version.join('.')} was found without venv/ensurepip. Install it:`); lines.push(' sudo apt install python3-venv (Debian/Ubuntu)'); } else { const old = seen.find((s) => s.version && !versionAtLeast(s.version, MIN_PYTHON)); const found = old ? ` (found ${old.version.join('.')})` : ''; - lines.push(` 2. Install Python 3.10 or newer${found}.`); + lines.push(` 2. Install Python 3.10 to 3.14${found}.`); } lines.push(' 3. Or install the CLI directly:'); lines.push(' curl -fsSL https://deepgram.com/install.sh | sh'); lines.push(' brew install deepgram/tap/deepgram'); - return `${lines.join('\n')}\n`; + return new LauncherError(`${lines.join('\n')}\n`); } // Decides how to run deepctl. Resolves to {command, args, strategy}, plus @@ -1049,18 +1146,26 @@ async function resolveRunner(argv, opts = {}) { } } - if (!forced || forced === 'uvx') { - const uv = findUv(opts); - if (uv) return { command: uv, args: ['tool', 'run', '--from', spec, 'deepctl', ...argv], strategy: 'uvx' }; - if (forced) throw new LauncherError('DEEPCTL_NPM_RUNNER=uvx, but uv was not found.'); + const uv = !forced || forced === 'uvx' ? findUv(opts) : null; + if (forced === 'uvx' && !uv) throw new LauncherError('DEEPCTL_NPM_RUNNER=uvx, but uv was not found.'); + const locked = lockedRequirements(env, version, opts); + const venvRunner = (layout, strategy) => { + const runner = { command: layout.deepctl, args: argv, strategy }; + if (layout.ephemeral) runner.cleanup = layout.dir; + return runner; + }; + + if (uv) { + // `uv tool run --with-requirements` does not check hashes, so with a + // requirements file uv builds a venv like the Python fallback does + // (in seconds, from uv's cache), through `uv pip install`, which does. + if (!locked) return { command: uv, args: ['tool', 'run', '--no-config', '--from', spec, 'deepctl', ...argv], strategy: 'uvx' }; + return venvRunner(await buildVenv({ installer: { uv }, key, spec, locked, version, opts }), 'uvx'); } const { python, seen } = await findPython(opts); - if (!python) throw new LauncherError(installHelp(seen)); - const layout = await buildVenv({ python, key, spec, version, opts }); - const runner = { command: layout.deepctl, args: argv, strategy: 'venv' }; - if (layout.ephemeral) runner.cleanup = layout.dir; - return runner; + if (!python) throw installHelp(seen); + return venvRunner(await buildVenv({ installer: { python }, key, spec, locked, version, opts }), 'venv'); } async function run(argv = process.argv.slice(2), opts = {}) { @@ -1203,10 +1308,12 @@ module.exports = { readOwner, writeOwner, mentionsDir, + uvConfigArgs, findPython, findUv, installHelp, lockIsStale, + lockedRequirements, packageVersion, parseProbeOutput, probePython, diff --git a/npm/package.json b/npm/package.json index 165efa9e..b545ff9b 100644 --- a/npm/package.json +++ b/npm/package.json @@ -9,7 +9,8 @@ }, "files": [ "bin/", - "lib/" + "lib/", + "requirements-*.txt" ], "engines": { "node": ">=20" diff --git a/npm/scripts/local_index.py b/npm/scripts/local_index.py new file mode 100644 index 00000000..fa214a84 --- /dev/null +++ b/npm/scripts/local_index.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Lay out wheels and sdists as a static package index, for tests and CI. + + python3 npm/scripts/local_index.py --out idx dist/ wheels/ + python3 -m http.server -d idx 8000 + +Writes a PEP 503 simple index (``idx/simple//``, for pip and uv) +and the slice of PyPI's JSON API that write_requirements.py reads +(``idx/pypi///json``), so the launcher's hash-locked +install can be exercised end to end without PyPI. Each file's +``upload_time_iso_8601`` is its modification time. +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import os +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def canonical(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def parse(filename: str) -> tuple[str, str, str] | None: + """(project, version, packagetype) from a wheel or sdist file name.""" + if filename.endswith(".whl"): + name, version = filename.split("-")[:2] + return canonical(name), version, "bdist_wheel" + if filename.endswith(".tar.gz"): + name, _, version = filename[: -len(".tar.gz")].rpartition("-") + return canonical(name), version, "sdist" + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n")[0]) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("sources", type=Path, nargs="+") + args = parser.parse_args() + + projects: dict[str, dict[str, list[dict[str, Any]]]] = {} + for source in args.sources: + for path in sorted(source.iterdir()): + parsed = parse(path.name) + if not parsed: + continue + project, version, packagetype = parsed + target = args.out / "simple" / project / path.name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + mtime = datetime.fromtimestamp(os.stat(path).st_mtime, timezone.utc) + projects.setdefault(project, {}).setdefault(version, []).append( + { + "filename": path.name, + "packagetype": packagetype, + "digests": { + "sha256": hashlib.sha256(path.read_bytes()).hexdigest() + }, + "upload_time_iso_8601": mtime.isoformat().replace("+00:00", "Z"), + "url": f"../../../simple/{project}/{path.name}", + } + ) + + for project, versions in projects.items(): + files = [f for entries in versions.values() for f in entries] + links = "\n".join( + f'' + f"{html.escape(f['filename'])}" + for f in files + ) + page = f"\n{links}\n\n" + (args.out / "simple" / project / "index.html").write_text(page) + for version, entries in versions.items(): + api = args.out / "pypi" / project / version / "json" + api.parent.mkdir(parents=True, exist_ok=True) + info = {"name": project, "version": version} + api.write_text(json.dumps({"info": info, "urls": entries})) + names = "\n".join(f'{p}' for p in sorted(projects)) + (args.out / "simple" / "index.html").write_text( + f"\n{names}\n\n" + ) + print(f"{len(projects)} projects in {args.out}") + + +if __name__ == "__main__": + main() diff --git a/npm/scripts/write_requirements.py b/npm/scripts/write_requirements.py new file mode 100644 index 00000000..0872ee8e --- /dev/null +++ b/npm/scripts/write_requirements.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Write npm/requirements-.txt, the only thing the launcher installs. + + python3 npm/scripts/write_requirements.py --dist dist/ --uploaded-since + +The release runs this after PyPI serves the version (verify-published), and +`npm publish` ships the result inside the package. The launcher installs +from it with ``--require-hashes --no-deps --only-binary :all:`` (pip or +``uv pip install``), so every line pins one distribution with its sha256 +hashes: + +* deepctl and every workspace package it depends on, at its workspace + version, with the hashes of the files PyPI serves for that version (its + JSON API, ``/pypi///json``). Not hashes of a local build: + wheels aren't byte-reproducible, and a version that was already on PyPI + (skip-existing) keeps its original files. +* every other dependency at its uv.lock version, with uv.lock's hashes + (``uv export --frozen --no-dev --no-emit-workspace``). + +``--dist`` cross-checks the release's own build. A file PyPI received +during this run (uploaded at or after ``--uploaded-since``) must match it +byte for byte; a file PyPI already had is kept as PyPI serves it, and the +differing rebuild is only reported. Without ``--uploaded-since``, any +difference fails. + +``--index-url`` points the JSON lookups at another index (tests and CI +serve one locally; see local_index.py). Needs uv on PATH and Python 3.11+ +(or tomli). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +import time +import urllib.request +from datetime import datetime +from pathlib import Path +from typing import Any + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - only taken on Python 3.10 + try: + import tomli as tomllib + except ModuleNotFoundError: + sys.exit("error: Python 3.11+ required (tomllib), or install tomli") + +ROOT = Path(__file__).resolve().parents[2] +NPM_DIR = ROOT / "npm" +EXPORT = ["uv", "export", "--frozen", "--no-dev", "--no-header", "--no-annotate"] +PYPI = "https://pypi.org" + + +def normalize(name: str) -> str: + """Distribution file-name form: lowercase, runs of -_. become _.""" + return re.sub(r"[-_.]+", "_", name).lower() + + +def canonical(name: str) -> str: + """Project-name form used in index URLs.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def uv_export(*extra: str) -> str: + return subprocess.run( + [*EXPORT, *extra], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout + + +def workspace_members() -> list[tuple[str, str, str]]: + """(name, version, marker) for deepctl and each workspace dependency.""" + members = [] + for line in uv_export("--no-hashes").splitlines(): + if not line.startswith("-e "): + continue + target, _, marker = line[3:].partition(";") + pyproject = tomllib.loads( + (ROOT / target.strip() / "pyproject.toml").read_text(encoding="utf-8") + ) + project = pyproject["project"] + members.append((project["name"], project["version"], marker.strip())) + if not members: + sys.exit("error: uv export listed no workspace packages") + return sorted(members) + + +def fetch_release(index: str, name: str, version: str) -> list[dict[str, Any]]: + """The files the index serves for name==version (its JSON API ``urls``).""" + url = f"{index.rstrip('/')}/pypi/{canonical(name)}/{version}/json" + for attempt in range(1, 4): + try: + with urllib.request.urlopen(url, timeout=30) as response: + files: list[dict[str, Any]] = json.load(response)["urls"] + return files + except Exception as err: # retried, then reported + if attempt == 3: + sys.exit(f"error: could not read {url}: {err}") + time.sleep(5 * attempt) + raise AssertionError("unreachable") + + +def parse_time(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def served_hashes( + files: list[dict[str, Any]], + name: str, + version: str, + dist: Path | None, + since: datetime | None, +) -> list[str]: + """sha256 of every file served for name==version, cross-checked with dist.""" + if not any(f.get("packagetype") == "bdist_wheel" for f in files): + sys.exit(f"error: the index serves no wheel for {name}=={version}") + served = {f["filename"]: f for f in files} + for path in local_files(dist, name, version) if dist else []: + entry = served.get(path.name) + if entry is None: + sys.exit(f"error: {path.name} was built but the index does not serve it") + local = hashlib.sha256(path.read_bytes()).hexdigest() + if local == entry["digests"]["sha256"]: + continue + uploaded = parse_time(entry["upload_time_iso_8601"]) + if since is None or uploaded >= since: + sys.exit( + f"error: {path.name} on the index (sha256 {entry['digests']['sha256']}) " + f"differs from the file this release built (sha256 {local})" + ) + print( + f"note: keeping the index's {path.name}, uploaded {uploaded:%Y-%m-%d} " + "before this run; the local rebuild differs", + file=sys.stderr, + ) + return [served[f]["digests"]["sha256"] for f in sorted(served)] + + +def local_files(dist: Path, name: str, version: str) -> list[Path]: + """Wheels and sdists in ``dist`` for name==version.""" + found = [] + for path in sorted(dist.iterdir()): + if path.name.endswith(".whl"): + dist_name, dist_version = path.name.split("-")[:2] + elif path.name.endswith(".tar.gz"): + dist_name, _, dist_version = path.name[: -len(".tar.gz")].rpartition("-") + else: + continue + if normalize(dist_name) == normalize(name) and dist_version == version: + found.append(path) + return found + + +def requirement(name: str, version: str, marker: str, hashes: list[str]) -> str: + head = f"{name}=={version}" + (f" ; {marker}" if marker else "") + return " \\\n".join([head, *(f" --hash=sha256:{h}" for h in hashes)]) + "\n" + + +def check(text: str, members: list[tuple[str, str, str]]) -> None: + """Every requirement carries a hash; every workspace pin is exact.""" + for block in re.split(r"\n(?=\S)", text.strip()): + if "--hash=sha256:" not in block: + sys.exit(f"error: no hash for: {block.splitlines()[0]}") + for name, version, _ in members: + if not re.search(rf"^{re.escape(name)}=={re.escape(version)}\b", text, re.M): + sys.exit(f"error: {name}=={version} is missing") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n")[0]) + parser.add_argument( + "--index-url", default=PYPI, help=f"JSON API base (default: {PYPI})" + ) + parser.add_argument( + "--dist", + type=Path, + help="the release's built wheels and sdists, to cross-check", + ) + parser.add_argument( + "--uploaded-since", + type=parse_time, + help="when this release run started (ISO 8601): files the index got later must match --dist", + ) + parser.add_argument( + "--out", + type=Path, + default=NPM_DIR, + help="directory to write into (default: npm/)", + ) + args = parser.parse_args(argv) + + root = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + version = root["project"]["version"] + package = json.loads((NPM_DIR / "package.json").read_text(encoding="utf-8")) + npm_version = package["version"] + if npm_version != version: + sys.exit(f"error: npm/package.json is {npm_version} but deepctl is {version}") + + members = workspace_members() + ours = "" + for name, member_version, marker in members: + files = fetch_release(args.index_url, name, member_version) + hashes = served_hashes( + files, name, member_version, args.dist, args.uploaded_since + ) + ours += requirement(name, member_version, marker, hashes) + text = ours + uv_export("--no-emit-workspace") + check(text, members) + out = args.out / f"requirements-{version}.txt" + out.write_text(text, encoding="utf-8", newline="\n") + print(f"wrote {out}: {len(members)} workspace packages", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index ff0fce02..b3908af7 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -49,6 +49,46 @@ function fakeSpawn(body) { const posixOnly = { skip: process.platform === 'win32' && 'POSIX shell scripts' }; +// A copy of the package as a release tarball has it: bin/, lib/, +// package.json and requirements-.txt, which the repository's npm/ +// directory does not have (the release writes it). `requirements: null` +// leaves the file out. +function packageCopy(requirements = 'deepctl==0.0.0 \\\n --hash=sha256:0000\n') { + const dir = tmpdir(); + for (const sub of ['bin', 'lib']) fs.cpSync(path.join(PKG_DIR, sub), path.join(dir, sub), { recursive: true }); + fs.copyFileSync(path.join(PKG_DIR, 'package.json'), path.join(dir, 'package.json')); + const version = launcher.packageVersion(PKG_DIR); + const requirementsFile = path.join(dir, `requirements-${version}.txt`); + if (requirements !== null) fs.writeFileSync(requirementsFile, requirements); + return { dir, bin: path.join(dir, 'bin', 'dg.js'), version, requirementsFile }; +} + +// A stand-in uv that logs its arguments to `log`. `uv venv DIR` lays down +// DIR/bin/python; `uv pip install --python PY ...` installs `deepctlBody` +// as deepctl next to PY, or fails with `pipExit` and a pip-like message. +function fakeUv(bin, deepctlBody, log, pipExit = 0) { + const template = path.join(bin, '..', 'deepctl-template'); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync(template, deepctlBody); + return fakeExe(bin, 'uv', [ + '#!/bin/sh', + `echo "$*" >> "${log}"`, + 'case "$1" in', + ' venv) for a; do d="$a"; done', + ' mkdir -p "$d/bin" && printf \'#!/bin/sh\\nexit 0\\n\' > "$d/bin/python" && chmod +x "$d/bin/python"; exit 0;;', + ' pip) prev=; for a; do [ "$prev" = --python ] && py="$a"; prev="$a"; done', + ` if [ ${pipExit} != 0 ]; then echo "error: Hash mismatch for deepctl" >&2; exit ${pipExit}; fi`, + ` cp "${template}" "$(dirname "$py")/deepctl" && chmod +x "$(dirname "$py")/deepctl"; exit 0;;`, + 'esac', + 'exit 3', + '', + ].join('\n')); +} + +function readLog(log) { + return fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\n') : []; +} + describe('package.json', () => { const pkg = JSON.parse(fs.readFileSync(path.join(PKG_DIR, 'package.json'), 'utf8')); @@ -183,7 +223,7 @@ describe('Python discovery', () => { assert.deepEqual(linux, ['/usr/local/bin/python3.11', '/usr/local/bin/python3', '/usr/bin/python3']); }); - test('Windows tries py -3.13..-3.10 and -3, then python, with WindowsApps last', () => { + test('Windows tries py -3.14..-3.10 and -3, then python, with WindowsApps last', () => { const present = new Set([ 'C:\\Windows\\py.exe', 'C:\\Users\\u\\AppData\\Local\\Microsoft\\WindowsApps\\python.exe', @@ -198,6 +238,7 @@ describe('Python discovery', () => { }; const got = launcher.pythonCandidates(opts).map((c) => [c.command, ...c.args].join(' ')); assert.deepEqual(got, [ + 'C:\\Windows\\py.exe -3.14', 'C:\\Windows\\py.exe -3.13', 'C:\\Windows\\py.exe -3.12', 'C:\\Windows\\py.exe -3.11', @@ -220,10 +261,10 @@ describe('Python discovery', () => { isExecutable: (f) => present.has(f), }; const got = launcher.pythonCandidates(opts).map((c) => [c.command, ...c.args].join(' ')); - assert.equal(got.length, 6); - assert.equal(got[0], `${apps}\\py.exe -3.13`); - assert.equal(got[4], `${apps}\\py.exe -3`); - assert.equal(got[5], `${apps}\\python.exe`); + assert.equal(got.length, 7); + assert.equal(got[0], `${apps}\\py.exe -3.14`); + assert.equal(got[5], `${apps}\\py.exe -3`); + assert.equal(got[6], `${apps}\\python.exe`); }); test('the version probe rejects a Store stub that prints no version', async () => { @@ -252,15 +293,33 @@ describe('Python discovery', () => { }); test('the help text names the fix for what was found', () => { - const noVenv = launcher.installHelp([{ version: [3, 12], ok: false, hasVenv: false }]); + const noVenv = launcher.installHelp([{ version: [3, 12], ok: false, hasVenv: false }]).message; assert.match(noVenv, /python3-venv/); assert.match(noVenv, /astral\.sh\/uv\/install\.sh/); assert.match(noVenv, /brew install deepgram\/tap\/deepgram/); assert.match(noVenv, /python\.bat shims and uv\.cmd are skipped/); - const old = launcher.installHelp([{ version: [3, 9], ok: false, hasVenv: true }]); + const old = launcher.installHelp([{ version: [3, 9], ok: false, hasVenv: true }]).message; assert.match(old, /found 3\.9/); assert.doesNotMatch(old, /python3-venv/); }); + + test('Python 3.15+ is not used: the lock has no wheels for it yet', async () => { + const probe = (version) => launcher.probePython({ command: 'py', args: [] }, async () => ({ status: 0, stdout: `(${version.join(', ')})` })); + assert.equal((await probe([3, 14])).ok, true); + assert.equal((await probe([3, 15])).ok, false); + const err = launcher.installHelp([{ version: [3, 15], ok: false, hasVenv: true }]); + assert.match(err.message, /^Python 3\.15 is too new for this release.*no Python 3\.10 to 3\.14/); + assert.match(err.hint, /Install Python 3\.12, or uv/); + }); + + test('the version probe runs isolated (-I), so the cwd cannot shadow venv or ensurepip', async () => { + let seen; + await launcher.probePython({ command: 'python3', args: [] }, async (cmd, args) => { + seen = args; + return { status: 0, stdout: '(3, 12)' }; + }); + assert.deepEqual(seen.slice(0, 2), ['-I', '-c']); + }); }); describe('cache directory', () => { @@ -344,7 +403,10 @@ describe('Windows never reuses a cached venv', () => { }); test('resolveRunner goes to uv instead of the cached venv', async () => { - const r = await launcher.resolveRunner(['--version'], { ...win, version: '1.0.0', isExecutable: (f) => f === 'C:\\bin\\uv.exe' }); + // The spec override skips the one-run build, which would write win32 + // paths to this filesystem; the next test covers that build. + const env = { ...win.env, DEEPCTL_NPM_PYPI_SPEC: 'deepctl==1.0.0' }; + const r = await launcher.resolveRunner(['--version'], { ...win, env, version: '1.0.0', isExecutable: (f) => f === 'C:\\bin\\uv.exe' }); assert.equal(r.strategy, 'uvx'); assert.equal(r.command, 'C:\\bin\\uv.exe'); }); @@ -362,13 +424,16 @@ describe('Windows never reuses a cached venv', () => { for (const f of ['.complete', 'bin/deepctl', 'bin/python']) fs.writeFileSync(path.join(root, '1.0.0', f), ''); const calls = []; const spawn = fakeSpawn((command, args) => { - calls.push(args.slice(-2).join(' ')); + calls.push(args); if (args.includes('venv')) fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); return 0; }); const lines = []; + const requirementsFile = path.join(home, 'requirements-1.0.0.txt'); + fs.writeFileSync(requirementsFile, ''); const r = await launcher.resolveRunner(['x'], { - env: { PATH: '/usr/bin', XDG_CACHE_HOME: path.join(home, 'c') }, + requirementsFile, + env: { PATH: '/usr/bin', XDG_CACHE_HOME: path.join(home, 'c'), PIP_INDEX_URL: 'http://idx/simple' }, version: '1.0.0', platform: 'linux', cacheReuse: false, @@ -382,7 +447,13 @@ describe('Windows never reuses a cached venv', () => { assert.equal(path.dirname(r.cleanup), root); assert.match(path.basename(r.cleanup), /^1\.0\.0-.{6}$/); assert.equal(r.command, path.join(r.cleanup, 'bin', 'deepctl')); - assert.equal(calls.length, 3); + assert.equal(calls.length, 2); + assert.deepEqual(calls[0].slice(0, 3), ['-I', '-m', 'venv']); + // pip installs only the release's pinned, hashed wheels. Its own config + // (pip.conf, PIP_*) applies; -I keeps cwd modules out. + assert.deepEqual(calls[1].slice(0, 4), ['-I', '-m', 'pip', 'install']); + assert.ok(!calls[1].includes('--isolated')); + assert.deepEqual(calls[1].slice(-6), ['--require-hashes', '--no-deps', '-r', requirementsFile, '--only-binary', ':all:']); assert.match(lines.join(''), /for this run.*install uv.*install\.ps1/); assert.equal(lines.join('').trimEnd().split('\n').length, 1, 'the hint is one line'); assert.equal(fs.existsSync(path.join(r.cleanup, '.complete')), false); @@ -390,6 +461,27 @@ describe('Windows never reuses a cached venv', () => { }); }); +describe("uv's own configuration", () => { + const files = (...present) => (f) => present.includes(f); + test('the user uv.toml is passed explicitly, else the machine one; a project one never is', () => { + const base = { platform: 'linux', home: '/home/u' }; + assert.deepEqual(launcher.uvConfigArgs({}, { ...base, configExists: files('/home/u/.config/uv/uv.toml', '/etc/uv/uv.toml') }), ['--config-file', '/home/u/.config/uv/uv.toml']); + assert.deepEqual(launcher.uvConfigArgs({ XDG_CONFIG_HOME: '/x' }, { ...base, configExists: files('/x/uv/uv.toml') }), ['--config-file', '/x/uv/uv.toml']); + assert.deepEqual(launcher.uvConfigArgs({}, { ...base, configExists: files('/etc/xdg/uv/uv.toml') }), ['--config-file', '/etc/xdg/uv/uv.toml']); + assert.deepEqual(launcher.uvConfigArgs({}, { ...base, configExists: files('/etc/uv/uv.toml') }), ['--config-file', '/etc/uv/uv.toml']); + assert.deepEqual(launcher.uvConfigArgs({}, { ...base, configExists: files('/home/u/project/uv.toml') }), []); + // UV_CONFIG_FILE is read by uv itself. + assert.deepEqual(launcher.uvConfigArgs({ UV_CONFIG_FILE: '/c.toml' }, { ...base, configExists: () => true }), []); + }); + + test('Windows: %APPDATA%, then %PROGRAMDATA%', () => { + const env = { APPDATA: 'C:\\Users\\u\\AppData\\Roaming', PROGRAMDATA: 'C:\\ProgramData' }; + const opts = { platform: 'win32', home: 'C:\\Users\\u' }; + assert.deepEqual(launcher.uvConfigArgs(env, { ...opts, configExists: files('C:\\ProgramData\\uv\\uv.toml') }), ['--config-file', 'C:\\ProgramData\\uv\\uv.toml']); + assert.deepEqual(launcher.uvConfigArgs(env, { ...opts, configExists: files('C:\\Users\\u\\AppData\\Roaming\\uv\\uv.toml', 'C:\\ProgramData\\uv\\uv.toml') }), ['--config-file', 'C:\\Users\\u\\AppData\\Roaming\\uv\\uv.toml']); + }); +}); + describe('pruning one-run venvs', () => { const HOUR = 60 * 60 * 1000; function aged(root, name, ageMs) { @@ -839,9 +931,74 @@ describe('resolveRunner', () => { await assert.rejects(launcher.resolveRunner([], { env: { DEEPCTL_NPM_RUNNER: 'pipx' }, version: '1.0.0' }), /must be one of/); }); - test('uses uv with the pinned spec when no venv is cached', async () => { + test('with no venv cached, uv builds one from the hash-locked requirements file', async () => { + const home = tmpdir(); + const cache = path.join(home, 'c'); + const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); + fs.writeFileSync(requirementsFile, 'deepctl==1.2.3 --hash=sha256:00\n'); + const calls = []; + const cwds = []; + const spawn = (command, args, options) => { + cwds.push(options.cwd); + return fakeSpawnBody(command, args); + }; + const fakeSpawnBody = fakeSpawn((command, args) => { + calls.push([command, ...args]); + if (args[0] === 'venv') fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + if (args[0] === 'pip') for (const f of ['python', 'deepctl']) fs.writeFileSync(path.join(path.dirname(args[args.indexOf('--python') + 1]), f), ''); + return 0; + }); const r = await launcher.resolveRunner(['--version'], { - env: { PATH: '/bin' }, + env: { PATH: '/bin', XDG_CACHE_HOME: cache }, + version: '1.2.3', + platform: 'linux', + home, + realpath: (f) => f, + isExecutable: (f) => f === '/bin/uv', + requirementsFile, + prepareRoot: (spec) => fs.mkdirSync(spec.dir, { recursive: true }) || true, + trustRoot: () => false, + spawn, + stderr: { write: () => {} }, + }); + const root = path.join(cache, 'deepgram-cli', 'venvs'); + const dir = path.join(root, '1.2.3'); + assert.deepEqual(calls, [ + ['/bin/uv', 'venv', '--no-config', '--quiet', '--allow-existing', '--python', '>=3.10,<3.15', dir], + ['/bin/uv', 'pip', 'install', '--no-config', '--quiet', '--python', path.join(dir, 'bin', 'python'), '--require-hashes', '--no-deps', '-r', requirementsFile, '--only-binary', ':all:'], + ]); + // Never the user's directory: a project's uv.toml there must not apply. + assert.deepEqual(cwds, [root, root]); + assert.deepEqual(r, { command: path.join(dir, 'bin', 'deepctl'), args: ['--version'], strategy: 'uvx' }); + assert.equal(fs.existsSync(path.join(dir, '.complete')), true); + }); + + test('a missing requirements file is refused before anything is installed', async () => { + let spawned = false; + await assert.rejects( + launcher.resolveRunner([], { + env: { PATH: '/bin' }, + version: '1.2.3', + platform: 'linux', + home: '/nonexistent', + realpath: (f) => f, + isExecutable: (f) => f === '/bin/uv', + exists: () => false, + isRequirementsFile: () => false, + spawn: () => (spawned = true), + }), + (err) => { + assert.match(err.message, /missing requirements-1\.2\.3\.txt/); + assert.match(err.hint, /npm install -g @deepgram\/cli@1\.2\.3/); + return true; + }, + ); + assert.equal(spawned, false); + }); + + test('DEEPCTL_NPM_PYPI_SPEC (testing) runs the spec through uv tool run, without a requirements file', async () => { + const r = await launcher.resolveRunner(['--version'], { + env: { PATH: '/bin', DEEPCTL_NPM_PYPI_SPEC: 'deepctl==1.2.3' }, version: '1.2.3', platform: 'linux', home: '/nonexistent', @@ -850,7 +1007,8 @@ describe('resolveRunner', () => { isExecutable: (f) => f === '/bin/uv', exists: () => false, }); - assert.deepEqual(r, { command: '/bin/uv', args: ['tool', 'run', '--from', 'deepctl==1.2.3', 'deepctl', '--version'], strategy: 'uvx' }); + assert.deepEqual(r, { command: '/bin/uv', args: ['tool', 'run', '--no-config', '--from', 'deepctl==1.2.3', 'deepctl', '--version'], strategy: 'uvx' }); + assert.equal(launcher.lockedRequirements({ DEEPCTL_NPM_PYPI_SPEC: 'x' }, '1.2.3'), null); }); test('a cached venv wins over uv', async () => { @@ -900,35 +1058,73 @@ describe('resolveRunner', () => { }); describe('bin/dg.js end to end', posixOnly, () => { - function runBin(env, args = []) { - return spawnSync(process.execPath, [BIN, ...args], { env, encoding: 'utf8' }); + function runCopy(pkg, env, args = []) { + return spawnSync(process.execPath, [pkg.bin, ...args], { env, encoding: 'utf8' }); } test('a dg run from inside deepctl (nested launcher) still works', () => { const home = tmpdir(); const bin = path.join(home, 'bin'); - fakeExe(bin, 'uv', '#!/bin/sh\necho nested-ok\n'); - const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_LAUNCHER_DEPTH: '1' }); + fakeUv(bin, '#!/bin/sh\necho nested-ok\n', path.join(home, 'uv.log')); + const r = runCopy(packageCopy(), { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_LAUNCHER_DEPTH: '1' }); assert.equal(r.status, 0); assert.equal(r.stdout, 'nested-ok\n'); }); test('no uv and no Python: clear error on stderr, nothing on stdout, exit 1', () => { const home = tmpdir(); - const r = runBin({ PATH: '/nonexistent', HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + const r = runCopy(packageCopy(), { PATH: '/nonexistent', HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); assert.equal(r.status, 1); assert.equal(r.stdout, ''); - assert.match(r.stderr, /^ERROR: The Deepgram CLI needs uv or Python 3\.10\+/); + assert.match(r.stderr, /^ERROR: The Deepgram CLI needs uv or Python 3\.10 to 3\.14/); }); - test('passes arguments, env and exit code through a fake uv', () => { + test('passes arguments, env and exit code through, from a venv uv built from the requirements file', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const log = path.join(home, 'uv.log'); + fakeUv(bin, '#!/bin/sh\necho "$DEEPCTL_LAUNCHER $*"\nexit 7\n', log); + const pkg = packageCopy(); + const env = { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }; + const r = runCopy(pkg, env, ['listen', 'a b.wav']); + assert.equal(r.status, 7, r.stderr); + assert.equal(r.stdout, 'npm listen a b.wav\n'); + const calls = readLog(log); + assert.equal(calls.length, 2); + assert.match(calls[1], new RegExp(`^pip install --no-config --quiet --python \\S+/bin/python --require-hashes --no-deps -r ${pkg.requirementsFile} --only-binary :all:$`)); + // The second run reuses the venv: uv is not called again. + assert.equal(runCopy(pkg, env, ['x']).stdout, 'npm x\n'); + assert.equal(readLog(log).length, 2); + }); + + test('without its requirements file the package installs nothing and says how to fix it', () => { const home = tmpdir(); const bin = path.join(home, 'bin'); - fakeExe(bin, 'uv', '#!/bin/sh\necho "$DEEPCTL_LAUNCHER $*"\nexit 7\n'); - const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }, ['listen', 'a b.wav']); - assert.equal(r.status, 7); - const version = launcher.packageVersion(PKG_DIR); - assert.equal(r.stdout, `npm tool run --from deepctl==${version} deepctl listen a b.wav\n`); + const log = path.join(home, 'uv.log'); + fakeUv(bin, '#!/bin/sh\necho ran\n', log); + const pkg = packageCopy(null); + const r = runCopy(pkg, { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + assert.equal(r.status, 1); + assert.equal(r.stdout, ''); + assert.equal(r.stderr, `ERROR: This @deepgram/cli package is missing requirements-${pkg.version}.txt, so dg can't install the CLI safely.\nHINT: Reinstall it: npm install -g @deepgram/cli@${pkg.version}\n`); + assert.deepEqual(readLog(log), []); + }); + + test('a download that fails its hash check: exit 1, nothing runs, no venv left', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const log = path.join(home, 'uv.log'); + const ran = path.join(home, 'ran'); + fakeUv(bin, `#!/bin/sh\ntouch "${ran}"\n`, log, 1); + const pkg = packageCopy(); + const env = { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }; + const r = runCopy(pkg, env); + assert.equal(r.status, 1); + assert.equal(r.stdout, ''); + assert.match(r.stderr, /Hash mismatch/); + assert.match(r.stderr, new RegExp(`ERROR: Installing deepctl==${pkg.version.replace(/\./g, '\\.')} failed \\(exit 1\\)\\.\\nHINT: .*hash mismatch above means a download did not match this release`)); + assert.equal(fs.existsSync(ran), false); + assert.deepEqual(fs.readdirSync(launcher.cacheRoots({ env, home })[0]), []); }); test('a SIGINT sent only to the launcher reaches the child after the grace period', async () => { @@ -974,6 +1170,7 @@ describe('bin/dg.js end to end', posixOnly, () => { // GitHub's ubuntu runners) wins and builds a real venv. const fakePython = [ '#!/bin/sh', + '[ "$1" = -I ] && shift', 'case "$1 $2" in', ' "-c "*) echo "(3, 12)"; exit 0;;', ` "-m venv") mkdir -p "$3/bin" && cp "${path.join(home, 'checker')}" "$3/bin/deepctl" && printf '#!/bin/sh\\nexit 0\\n' > "$3/bin/python" && chmod +x "$3/bin/deepctl" "$3/bin/python"; exit 0;;`, @@ -986,7 +1183,7 @@ describe('bin/dg.js end to end', posixOnly, () => { } fs.writeFileSync(path.join(home, 'checker'), checker); const { spawn } = require('node:child_process'); - const child = spawn(process.execPath, [BIN], { + const child = spawn(process.execPath, [packageCopy().bin], { env: { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }, stdio: ['pipe', 'pipe', 'pipe'], }); @@ -1004,8 +1201,8 @@ describe('bin/dg.js end to end', posixOnly, () => { test('re-raises the signal the child died from', () => { const home = tmpdir(); const bin = path.join(home, 'bin'); - fakeExe(bin, 'uv', '#!/bin/sh\nkill -TERM $$\n'); - const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + fakeUv(bin, '#!/bin/sh\nkill -TERM $$\n', path.join(home, 'uv.log')); + const r = runCopy(packageCopy(), { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); assert.equal(r.status, null); assert.equal(r.signal, 'SIGTERM'); assert.doesNotMatch(r.stderr, /crashed/); @@ -1014,18 +1211,18 @@ describe('bin/dg.js end to end', posixOnly, () => { test('names the signal when the child crashes', { skip: process.platform === 'win32' && 'POSIX signals' }, () => { const home = tmpdir(); const bin = path.join(home, 'bin'); - fakeExe(bin, 'uv', '#!/bin/sh\nkill -SEGV $$\n'); - const r = runBin({ PATH: bin, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); + fakeUv(bin, '#!/bin/sh\nkill -SEGV $$\n', path.join(home, 'uv.log')); + const r = runCopy(packageCopy(), { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c') }); assert.equal(r.signal, 'SIGSEGV'); - assert.match(r.stderr, /^ERROR: .*uv crashed \(SIGSEGV\)\./m); + assert.match(r.stderr, /^ERROR: .*\/bin\/deepctl crashed \(SIGSEGV\)\./m); }); test('a venv build step killed by a signal says so', { skip: process.platform === 'win32' && 'POSIX signals' }, () => { const home = tmpdir(); const bin = path.join(home, 'bin'); - const fakePython = '#!/bin/sh\ncase "$1 $2" in\n "-c "*) echo "(3, 12)"; exit 0;;\nesac\nkill -KILL $$\n'; + const fakePython = '#!/bin/sh\n[ "$1" = -I ] && shift\ncase "$1 $2" in\n "-c "*) echo "(3, 12)"; exit 0;;\nesac\nkill -KILL $$\n'; for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) fakeExe(bin, name, fakePython); - const r = runBin({ PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }); + const r = runCopy(packageCopy(), { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }); assert.equal(r.status, 1); assert.match(r.stderr, /failed \(killed by SIGKILL\)/); }); @@ -1038,6 +1235,7 @@ describe('bin/dg.js end to end', posixOnly, () => { // until the test signals the launcher. describe('a signal during first-run setup', posixOnly, () => { const version = launcher.packageVersion(PKG_DIR); + const pkg = packageCopy(); // Dead, or a zombie nobody has reaped yet (PID 1 in a container). function gone(pid) { @@ -1078,15 +1276,13 @@ describe('a signal during first-run setup', posixOnly, () => { // pip leaves build subprocesses, and records both pids. const fakePython = [ '#!/bin/sh', + '[ "$1" = -I ] && shift', `hang() { sleep 300 & echo "$$ $!" > "${pidfile}.tmp"; mv "${pidfile}.tmp" "${pidfile}"; wait; exit 0; }`, 'case "$1 $2" in', ' "-c "*) echo "(3, 12)"; exit 0;;', ' "-m venv") mkdir -p "$3/bin" && cp "$0" "$3/bin/python" && printf \'#!/bin/sh\\nexit 0\\n\' > "$3/bin/deepctl" && chmod +x "$3/bin/deepctl"', ' [ "$HANG_AT" = venv ] && hang; exit 0;;', - ' "-m pip") case "$*" in', - ' *" -U pip"*) [ "$HANG_AT" = pip-upgrade ] && hang;;', - ' *) [ "$HANG_AT" = pip-install ] && hang;;', - ' esac; exit 0;;', + ' "-m pip") [ "$HANG_AT" = pip-install ] && hang; exit 0;;', 'esac', 'exit 0', '', @@ -1102,7 +1298,7 @@ describe('a signal during first-run setup', posixOnly, () => { function start(env) { const { spawn } = require('node:child_process'); - const child = spawn(process.execPath, [BIN], { env, stdio: ['ignore', 'pipe', 'pipe'] }); + const child = spawn(process.execPath, [pkg.bin], { env, stdio: ['ignore', 'pipe', 'pipe'] }); let stderr = ''; child.stderr.on('data', (d) => (stderr += d)); child.stdout.resume(); @@ -1110,7 +1306,7 @@ describe('a signal during first-run setup', posixOnly, () => { return { child, exited, stderr: () => stderr }; } - for (const step of ['venv', 'pip-upgrade', 'pip-install']) { + for (const step of ['venv', 'pip-install']) { test(`SIGTERM while the ${step} step runs: no orphan, no lock, no partial venv, dies by SIGTERM`, async () => { const ws = workspace(step); const run = start(ws.env); diff --git a/tests/unit/test_npm_requirements.py b/tests/unit/test_npm_requirements.py new file mode 100644 index 00000000..9cd474f5 --- /dev/null +++ b/tests/unit/test_npm_requirements.py @@ -0,0 +1,152 @@ +"""npm/scripts/write_requirements.py: workspace hashes come from the index.""" + +from __future__ import annotations + +import functools +import hashlib +import importlib.util +import json +import threading +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parents[2] / "npm" / "scripts" / "write_requirements.py" +) + + +@pytest.fixture +def wr(): + spec = importlib.util.spec_from_file_location("write_requirements", SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def index(tmp_path): + """A static JSON API served over HTTP; returns (base_url, publish).""" + root = tmp_path / "idx" + root.mkdir() + + def publish(name: str, version: str, files: dict[str, bytes], uploaded: str): + entries = [ + { + "filename": fn, + "packagetype": "bdist_wheel" if fn.endswith(".whl") else "sdist", + "digests": {"sha256": hashlib.sha256(data).hexdigest()}, + "upload_time_iso_8601": uploaded, + } + for fn, data in files.items() + ] + api = root / "pypi" / name / version / "json" + api.parent.mkdir(parents=True, exist_ok=True) + api.write_text(json.dumps({"urls": entries})) + + handler = functools.partial(SimpleHTTPRequestHandler, directory=str(root)) + handler.log_message = lambda *a: None # type: ignore[attr-defined] + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_address[1]}", publish + server.shutdown() + + +WHEEL = "deepctl_cmd_billing-0.1.0-py3-none-any.whl" +PYPI_BYTES = b"the wheel PyPI has served since 2025" +REBUILT = b"the same source, rebuilt today with new mtimes" + + +def run(wr, monkeypatch, tmp_path, url, *extra): + monkeypatch.setattr( + wr, "workspace_members", lambda: [("deepctl-cmd-billing", "0.1.0", "")] + ) + monkeypatch.setattr( + wr, "uv_export", lambda *a: "click==8.1.7 \\\n --hash=sha256:aa\n" + ) + out = tmp_path / "out" + out.mkdir(exist_ok=True) + # The repository's root and npm versions agree; only --out is redirected. + wr.main(["--index-url", url, "--out", str(out), *extra]) + return next(out.iterdir()).read_text() + + +def test_hashes_come_from_the_index_not_a_local_build(wr, index, monkeypatch, tmp_path): + url, publish = index + publish("deepctl-cmd-billing", "0.1.0", {WHEEL: PYPI_BYTES}, "2025-01-01T00:00:00Z") + text = run(wr, monkeypatch, tmp_path, url) + assert f"--hash=sha256:{hashlib.sha256(PYPI_BYTES).hexdigest()}" in text + assert text.startswith("deepctl-cmd-billing==0.1.0 \\\n") + + +def test_unchanged_sibling_rebuilt_differently_keeps_the_index_hash( + wr, index, monkeypatch, tmp_path, capsys +): + """A version already on the index (skip-existing) whose local rebuild differs.""" + url, publish = index + publish("deepctl-cmd-billing", "0.1.0", {WHEEL: PYPI_BYTES}, "2025-01-01T00:00:00Z") + dist = tmp_path / "dist" + dist.mkdir() + (dist / WHEEL).write_bytes(REBUILT) + text = run( + wr, + monkeypatch, + tmp_path, + url, + "--dist", + str(dist), + "--uploaded-since", + "2026-10-31T00:00:00Z", + ) + assert hashlib.sha256(PYPI_BYTES).hexdigest() in text + assert hashlib.sha256(REBUILT).hexdigest() not in text + assert "keeping the index's" in capsys.readouterr().err + + +def test_a_file_uploaded_by_this_run_must_match_the_build( + wr, index, monkeypatch, tmp_path +): + url, publish = index + publish("deepctl-cmd-billing", "0.1.0", {WHEEL: PYPI_BYTES}, "2026-10-31T12:00:00Z") + dist = tmp_path / "dist" + dist.mkdir() + (dist / WHEEL).write_bytes(REBUILT) + with pytest.raises(SystemExit, match="differs from the file this release built"): + run( + wr, + monkeypatch, + tmp_path, + url, + "--dist", + str(dist), + "--uploaded-since", + "2026-10-31T00:00:00Z", + ) + # Without --uploaded-since every difference fails. + with pytest.raises(SystemExit, match="differs"): + run(wr, monkeypatch, tmp_path, url, "--dist", str(dist)) + + +def test_a_built_file_the_index_does_not_serve_fails(wr, index, monkeypatch, tmp_path): + url, publish = index + publish("deepctl-cmd-billing", "0.1.0", {WHEEL: PYPI_BYTES}, "2025-01-01T00:00:00Z") + dist = tmp_path / "dist" + dist.mkdir() + (dist / "deepctl_cmd_billing-0.1.0.tar.gz").write_bytes(b"sdist") + with pytest.raises(SystemExit, match="does not serve it"): + run(wr, monkeypatch, tmp_path, url, "--dist", str(dist)) + + +def test_no_wheel_on_the_index_fails(wr, index, monkeypatch, tmp_path): + url, publish = index + publish( + "deepctl-cmd-billing", + "0.1.0", + {"deepctl_cmd_billing-0.1.0.tar.gz": b"s"}, + "2025-01-01T00:00:00Z", + ) + with pytest.raises(SystemExit, match="no wheel"): + run(wr, monkeypatch, tmp_path, url) From 41b1127123ec9a2f96d72572ea1b002714c66239 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:13:19 -0400 Subject: [PATCH 08/15] fix(npm): keep venvs and uv's cache in directories only the user controls A cache root is reused only when rootTrusted shows it is ours: a real directory, not a symlink, owned by this uid and not writable by group or others. When no root passes, the launcher used to stop with "No writable cache directory". It now builds a venv for that run in a fresh 0700 directory under the system temp directory, whose ancestors get the same ownership check, deletes it afterwards, and says on stderr which directory it would not trust and how to fix it. Nothing is written into the untrusted root. uv installs the files it extracted into its cache without hashing them again, and its default cache ($XDG_CACHE_HOME/uv) can sit in a directory others can write to. So uv always gets a launcher-owned cache: next to the venvs under the trusted root, inside the run's own directory (removed after setup, with pip running cacheless) when no root is trusted, and next to the one-run venvs on Windows. An existing cache directory that another user owns, or that group or others can write to, is not used: that run gets a one-run venv, and the hint says to remove the directory. A UV_CACHE_DIR the user set is left alone. Windows never reuses a venv. Its one-run venvs go under %LOCALAPPDATA%\deepgram-cli\venvs, and the directories the launcher creates there must be plain directories, not symlinks or junctions, before anything is built in them. --- npm/README.md | 10 +- npm/lib/launcher.js | 110 ++++++++++++++++--- npm/test/launcher.test.js | 219 +++++++++++++++++++++++++++++++++++++- 3 files changed, 323 insertions(+), 16 deletions(-) diff --git a/npm/README.md b/npm/README.md index 33a00baa..71ebd912 100644 --- a/npm/README.md +++ b/npm/README.md @@ -52,7 +52,15 @@ working: `UV_CONFIG_FILE`, and one `uv.toml`: your user file (`~/.config/uv/uv.toml`, or `%APPDATA%\uv\uv.toml` on Windows) if it exists, otherwise the system file (`/etc/uv/uv.toml`, or - `%PROGRAMDATA%\uv\uv.toml`). + `%PROGRAMDATA%\uv\uv.toml`). uv's `cache-dir` setting is the exception: + the launcher gives uv a cache of its own next to its environments, since uv + installs cached files without re-checking them, unless you set + `UV_CACHE_DIR`. + +The launcher reuses that cache only when you own it, nobody else can write to +it, and it isn't a symlink. If no cache directory qualifies, each run builds a +fresh environment in a private temporary directory and deletes it afterwards, +and says why on stderr. On Windows, step 1 is skipped: the launcher can't verify who controls a cache directory there, so it never runs an environment left by an earlier run. Every diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 67d2021d..9aa5b2f2 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -543,6 +543,27 @@ function ancestorsSafe(start, uid, gid, lstat, allowLinks) { } } +// A directory of ours for uv's cache: created 0700 if missing. One that +// already exists is refused if it is a link (a junction on Windows) rather +// than a plain directory, or, on POSIX, if another user owns it or group or +// others can write to it: it may have been prepared by someone else, so the +// fix is to remove it, not to chmod it. +function ownedDir(dir, opts = {}) { + const refuse = (why) => { + const err = new LauncherError(`${dir} ${why}.`, `Remove ${dir} and run dg again; dg recreates it.`); + err.dir = dir; + return err; + }; + mkdirPrivate(dir); + const st = (opts.lstat || fs.lstatSync)(dir); + if (st.isSymbolicLink() || !st.isDirectory()) throw refuse('is a link, not a plain directory'); + if (!isWindows(opts.platform || process.platform)) { + if (st.uid !== currentUid(opts)) throw refuse('belongs to another user'); + if ((st.mode & 0o022) !== 0) throw refuse('is writable by group or others'); + } + return dir; +} + function mkdirPrivate(dir) { try { fs.mkdirSync(dir, { mode: 0o700 }); @@ -906,26 +927,70 @@ function pruneSiblings(root, keep, now = Date.now()) { } } +// Where a one-run venv goes: a parent directory, and whether leftovers from +// crashed runs should be pruned there. +// +// Windows (no cache reuse at all): %LOCALAPPDATA%\deepgram-cli\venvs. Node +// cannot read its owner or ACLs, but it can see links: the two levels the +// launcher creates must be plain directories, not symlinks or junctions +// (lstat reports both as links), so nothing there can redirect the build +// into a directory someone else controls. The venv inside gets a fresh, +// unpredictable name. +// +// POSIX, when no cache root passed rootTrusted (`untrusted`): the system +// temp directory, held to the ancestor check a cache root gets (each level +// root's or ours, sticky if others can write to it), so another user can +// neither predict nor swap the 0700 directory mkdtemp creates there. +function oneRunBase(untrusted, opts = {}) { + const lstat = opts.lstat || fs.lstatSync; + if (untrusted) { + const tmp = (opts.realpath || fs.realpathSync)(opts.tmpdir || os.tmpdir()); + const uid = currentUid(opts); + const gid = opts.gid !== undefined ? opts.gid : process.getgid ? process.getgid() : null; + if (!ancestorsSafe(tmp, uid, gid, lstat, false)) throw new Error(`${tmp} is not safe from other users`); + return { base: tmp, prefix: 'deepgram-cli-', prune: false }; + } + const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; + const base = cacheRootSpecs(opts)[0].dir; + (opts.mkdir || fs.mkdirSync)(base, { recursive: true, mode: 0o700 }); + for (const dir of [p.dirname(base), base]) { + const st = lstat(dir); + if (st.isSymbolicLink() || !st.isDirectory()) throw new Error(`${dir} is a link or junction, not a plain directory`); + } + return { base, prefix: '', prune: true }; +} + // Builds a venv for this run only, in a directory created fresh with an // unpredictable name, so nothing another process prepared can be picked up. // The caller deletes it once deepctl exits. -async function buildPrivateVenv({ installer, key, spec, locked, version, opts = {} }) { +async function buildPrivateVenv({ installer, key, spec, locked, version, untrusted, hint, opts = {} }) { const stderr = opts.stderr || fdStderr; const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; - const base = cacheRootSpecs(opts)[0].dir; let dir; + let uvCache; try { - fs.mkdirSync(base, { recursive: true, mode: 0o700 }); - pruneOneRunVenvs(base, p); - dir = fs.mkdtempSync(p.join(base, `${key}-`)); + const { base, prefix, prune } = oneRunBase(untrusted, opts); + if (prune) pruneOneRunVenvs(base, p); + dir = fs.mkdtempSync(p.join(base, `${prefix}${key}-`)); + // Windows: a uv cache kept beside the one-run venvs, held to the same + // plain-directory check (oneRunBase), so runs after the first are quick. + if (!untrusted && installer.uv) uvCache = ownedDir(p.join(p.dirname(base), 'uv-cache'), opts); } catch (err) { - throw new LauncherError(`No writable directory for the Deepgram CLI: ${err.message}`, 'Install uv instead: it needs no setup on each run.'); + throw new LauncherError(`No safe, writable directory to set up the Deepgram CLI in: ${err.message}`, untrusted ? 'Point TMPDIR at a directory only you can write to.' : 'Point LOCALAPPDATA at a directory only you can write to.'); } const layout = venvLayout(dir, opts.platform); - if (installer.uv) stderr.write(`Setting up Deepgram CLI ${version} for this run...\n`); + if (untrusted) { + stderr.write(`Setting up Deepgram CLI ${version} for this run only: dg reuses a cache only from a directory you own that others can't write to, and ${untrusted} isn't one.\n`); + const elsewhere = (opts.platform || process.platform) === 'darwin' ? '' : ', or set XDG_CACHE_HOME to such a directory'; + stderr.write(hint ? `HINT: ${hint}\n` : `HINT: Make it yours and not group- or world-writable (chmod go-w)${elsewhere}.\n`); + } else if (installer.uv) stderr.write(`Setting up Deepgram CLI ${version} for this run...\n`); else stderr.write(`Setting up Deepgram CLI ${version} for this run (~1 min each time; install uv to speed it up: ${UV_INSTALL_WINDOWS})...\n`); + // No trusted root: uv's cache for this run lives inside the run's own + // directory and goes once setup is done; pip runs without a cache. + const privateCache = untrusted ? p.join(dir, '.setup-cache') : undefined; try { - await installInto(installer, layout, spec, locked, opts, { cwd: dir }); + await installInto(installer, layout, spec, locked, opts, { cwd: dir, uvCache: privateCache || uvCache, noPipCache: Boolean(untrusted) }); + if (privateCache) fs.rmSync(privateCache, { recursive: true, force: true }); } catch (err) { fs.rmSync(dir, { recursive: true, force: true }); throw err; @@ -1008,7 +1073,11 @@ async function installInto(installer, layout, spec, locked, opts, where = {}) { const what = `Installing ${spec}`; const hint = locked ? INSTALL_HINT : NETWORK_HINT; if (installer.uv) { - const cache = where.privateCache ? ['--cache-dir', where.privateCache] : []; + // uv installs the files it extracted into its cache without hashing them + // again, so its cache must be as private as the venv: never uv's default + // ($XDG_CACHE_HOME/uv may be group-writable), always one the launcher + // owns, unless the user chose one with UV_CACHE_DIR. + const cache = where.uvCache && !env.UV_CACHE_DIR ? ['--cache-dir', where.uvCache] : []; const config = uvConfigArgs(env, opts); const uv = (sub) => [...sub, '--no-config', ...config, '--quiet', ...cache]; await runStep(installer.uv, [...uv(['venv']), '--allow-existing', '--python', PYTHON_RANGE, layout.dir], 'Creating the Python venv', UV_VENV_HINT, opts, where.cwd); @@ -1017,7 +1086,7 @@ async function installInto(installer, layout, spec, locked, opts, where = {}) { } const { python } = installer; await runStep(python.command, [...python.args, '-I', '-m', 'venv', layout.dir], 'Creating the Python venv', VENV_HINT, opts, where.cwd); - const noCache = where.privateCache ? ['--no-cache-dir'] : []; + const noCache = where.noPipCache ? ['--no-cache-dir'] : []; const pip = [layout.python, '-I', '-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...noCache, ...source]; await runStep(pip[0], pip.slice(1), what, hint, opts, where.cwd); } @@ -1031,7 +1100,22 @@ async function buildVenv({ installer, key, spec, locked, version, opts = {} }) { const prepare = opts.prepareRoot || ((spec) => prepareRoot(spec, opts)); const rootSpec = cacheRootSpecs(opts).find(prepare); const root = rootSpec && rootSpec.dir; - if (!root) throw new LauncherError('No writable cache directory for the Deepgram CLI.', 'Set XDG_CACHE_HOME to a writable directory, or install uv.'); + // No root we can show is ours (another owner, writable by others, a + // symlink, or not writable at all): never reuse or populate one; build a + // venv for this run in a fresh private directory instead. + if (!root) return buildPrivateVenv({ installer, key, spec, locked, version, untrusted: cacheRootSpecs(opts)[0].dir, opts }); + // uv's cache sits beside the venvs, under the root rootTrusted passed. If + // an existing one is not ours alone, nothing in it can be trusted either. + let uvCache; + if (installer.uv) { + try { + uvCache = ownedDir(path.join(path.dirname(root), 'uv-cache'), opts); + } catch (err) { + const dir = err.dir || path.join(path.dirname(root), 'uv-cache'); + const hint = err instanceof LauncherError ? err.hint : `Remove ${dir} and run dg again; dg recreates it.`; + return buildPrivateVenv({ installer, key, spec, locked, version, untrusted: dir, hint, opts }); + } + } // Built in place: venv console scripts carry absolute shebangs, so a venv // built elsewhere and renamed into place would not run. const layout = venvLayout(path.join(root, key), opts.platform); @@ -1047,7 +1131,7 @@ async function buildVenv({ installer, key, spec, locked, version, opts = {} }) { fs.rmSync(layout.dir, { recursive: true, force: true }); stderr.write(`Setting up Deepgram CLI ${version} (one-time${installer.uv ? '' : ', ~1 min'})...\n`); try { - await installInto(installer, layout, spec, locked, opts, { cwd: root }); + await installInto(installer, layout, spec, locked, opts, { cwd: root, uvCache }); fs.writeFileSync(layout.marker, `${spec}\n`); } catch (err) { fs.rmSync(layout.dir, { recursive: true, force: true }); @@ -1309,6 +1393,8 @@ module.exports = { writeOwner, mentionsDir, uvConfigArgs, + oneRunBase, + ownedDir, findPython, findUv, installHelp, diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index b3908af7..95b24534 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -659,6 +659,139 @@ describe('cache root ownership (POSIX)', () => { }); }); +describe('no trusted cache root: a one-run venv instead', () => { + // Fake uv steps: `venv DIR` makes DIR/bin, `pip install --python PY` + // puts deepctl and python next to PY. + function uvSpawn(calls) { + return fakeSpawn((command, args) => { + calls.push(args); + if (args[0] === 'venv') fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + if (args[0] === 'pip') for (const f of ['python', 'deepctl']) fs.writeFileSync(path.join(path.dirname(args[args.indexOf('--python') + 1]), f), ''); + return 0; + }); + } + + // A complete-looking venv planted in each cache root; both roots are + // made untrusted by `spoil`. Returns what resolveRunner did. + async function runWith(spoil) { + const home = tmpdir(); + const tmp = tmpdir(); + const cache = path.join(home, 'c'); + const env = { PATH: '/bin', XDG_CACHE_HOME: cache }; + const specs = launcher.cacheRootSpecs({ env, home, tmpdir: tmp, platform: 'linux' }); + for (const spec of specs) { + fs.mkdirSync(path.join(spec.dir, '1.0.0', 'bin'), { recursive: true, mode: 0o700 }); + for (const f of ['.complete', 'bin/deepctl', 'bin/python']) fs.writeFileSync(path.join(spec.dir, '1.0.0', f), ''); + } + spoil(specs, home); + const requirementsFile = path.join(home, 'requirements-1.0.0.txt'); + fs.writeFileSync(requirementsFile, ''); + const calls = []; + const lines = []; + const r = await launcher.resolveRunner(['x'], { + env, + home, + tmpdir: tmp, + version: '1.0.0', + platform: 'linux', + isExecutable: (f) => f === '/bin/uv', + requirementsFile, + spawn: uvSpawn(calls), + stderr: { write: (t) => lines.push(t) }, + }); + return { r, calls, stderr: lines.join(''), tmp, specs }; + } + + function assertOneRun({ r, calls, stderr, tmp, specs }) { + // Built fresh under the temp directory, not in (or from) either root. + assert.equal(path.dirname(r.cleanup), fs.realpathSync(tmp)); + assert.match(path.basename(r.cleanup), /^deepgram-cli-1\.0\.0-.{6}$/); + assert.equal(fs.statSync(r.cleanup).mode & 0o777, 0o700); + assert.equal(r.command, path.join(r.cleanup, 'bin', 'deepctl')); + assert.equal(calls.length, 2); + // Hash-locked, wheels only, and uv's cache private to this run (an + // untrusted root may hold uv's cache too); the cache is gone afterwards. + const req = calls[1][calls[1].indexOf('-r') + 1]; + assert.deepEqual(calls[1].slice(-6), ['--require-hashes', '--no-deps', '-r', req, '--only-binary', ':all:']); + for (const c of calls) assert.equal(c[c.indexOf('--cache-dir') + 1], path.join(r.cleanup, '.setup-cache')); + assert.equal(fs.existsSync(path.join(r.cleanup, '.setup-cache')), false); + for (const spec of specs) assert.deepEqual(fs.readdirSync(spec.dir), ['1.0.0'], 'nothing written into an untrusted root'); + assert.match(stderr, new RegExp(`for this run only: .*${specs[0].dir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} isn't one\\.\\nHINT: `)); + } + + test('group-writable roots are not reused or written to', posixOnly, async () => { + const out = await runWith((specs) => { + for (const spec of specs) fs.chmodSync(path.dirname(spec.dir), 0o770); + }); + assertOneRun(out); + }); + + test('symlinked roots are not reused or written to', posixOnly, async () => { + const out = await runWith((specs, home) => { + for (const [i, spec] of specs.entries()) { + const top = path.dirname(spec.dir); + const real = path.join(home, `real-${i}`); + fs.renameSync(top, real); + fs.symlinkSync(real, top); + } + }); + // readdir follows the links: the planted venvs are still all there is. + assertOneRun(out); + }); + + test('roots owned by another user are not reused (lstat stand-in)', async () => { + const st = (uid, mode = 0o40700) => ({ uid, gid: uid, mode, isDirectory: () => true, isSymbolicLink: () => false }); + const opts = { platform: 'linux', uid: 1000, gid: 1000, realpath: (f) => f }; + for (const shared of [false, true]) { + const spec = { dir: '/h/.cache/deepgram-cli/venvs', shared }; + assert.equal(launcher.rootTrusted(spec, { ...opts, lstat: (d) => (d.startsWith('/h/.cache/deepgram-cli') ? st(2000) : st(0, 0o40755)) }), false); + } + }); + + test('the temp directory must itself be safe from other users', () => { + const st = (uid, mode) => ({ uid, gid: uid, mode, isDirectory: () => true, isSymbolicLink: () => false }); + const base = { platform: 'linux', uid: 1000, gid: 1000, realpath: (f) => f, tmpdir: '/scratch' }; + // World-writable without the sticky bit: anyone could rename our directory. + assert.throws(() => launcher.oneRunBase('/h/.cache', { ...base, lstat: (d) => (d === '/scratch' ? st(0, 0o40777) : st(0, 0o40755)) }), /not safe/); + // /tmp-style sticky 1777 is fine. + assert.equal(launcher.oneRunBase('/h/.cache', { ...base, lstat: (d) => (d === '/scratch' ? st(0, 0o41777) : st(0, 0o40755)) }).base, '/scratch'); + }); + + test('Windows: a junction or symlink where the one-run venvs go is refused', async () => { + const lstatWith = (linked) => (d) => ({ + isDirectory: () => d !== linked, + isSymbolicLink: () => d === linked, + }); + const win = { + env: { LOCALAPPDATA: 'C:\\Users\\u\\AppData\\Local' }, + platform: 'win32', + home: 'C:\\Users\\u', + mkdir: () => {}, + }; + const top = 'C:\\Users\\u\\AppData\\Local\\deepgram-cli'; + for (const linked of [top, `${top}\\venvs`]) { + assert.throws(() => launcher.oneRunBase(undefined, { ...win, lstat: lstatWith(linked) }), /link or junction/); + } + assert.deepEqual(launcher.oneRunBase(undefined, { ...win, lstat: lstatWith(null) }), { base: `${top}\\venvs`, prefix: '', prune: true }); + let spawned = false; + await assert.rejects( + launcher.resolveRunner([], { + ...win, + env: { ...win.env, Path: 'C:\\bin' }, + version: '1.0.0', + pkgDir: 'C:\\npm\\node_modules\\@deepgram\\cli', + realpath: (f) => f, + isExecutable: (f) => f === 'C:\\bin\\uv.exe', + isRequirementsFile: () => true, + lstat: lstatWith(top), + spawn: () => (spawned = true), + }), + /No safe, writable directory.*link or junction/, + ); + assert.equal(spawned, false); + }); +}); + describe('pruning other versions', () => { const DAY = 24 * 60 * 60 * 1000; function venv(root, name, { markerAgeDays, dirAgeMs } = {}) { @@ -963,16 +1096,96 @@ describe('resolveRunner', () => { }); const root = path.join(cache, 'deepgram-cli', 'venvs'); const dir = path.join(root, '1.2.3'); + // uv's cache is the launcher's own, beside the venvs, never uv's default. + const uvCache = path.join(cache, 'deepgram-cli', 'uv-cache'); assert.deepEqual(calls, [ - ['/bin/uv', 'venv', '--no-config', '--quiet', '--allow-existing', '--python', '>=3.10,<3.15', dir], - ['/bin/uv', 'pip', 'install', '--no-config', '--quiet', '--python', path.join(dir, 'bin', 'python'), '--require-hashes', '--no-deps', '-r', requirementsFile, '--only-binary', ':all:'], + ['/bin/uv', 'venv', '--no-config', '--quiet', '--cache-dir', uvCache, '--allow-existing', '--python', '>=3.10,<3.15', dir], + ['/bin/uv', 'pip', 'install', '--no-config', '--quiet', '--cache-dir', uvCache, '--python', path.join(dir, 'bin', 'python'), '--require-hashes', '--no-deps', '-r', requirementsFile, '--only-binary', ':all:'], ]); + assert.equal(fs.statSync(uvCache).mode & 0o777, 0o700); // Never the user's directory: a project's uv.toml there must not apply. assert.deepEqual(cwds, [root, root]); assert.deepEqual(r, { command: path.join(dir, 'bin', 'deepctl'), args: ['--version'], strategy: 'uvx' }); assert.equal(fs.existsSync(path.join(dir, '.complete')), true); }); + test('an existing uv cache that is not ours alone is refused, with a hint to remove it', () => { + const dir = tmpdir(); + const st = (uid, mode) => () => ({ uid, mode, isDirectory: () => true, isSymbolicLink: () => false }); + const opts = { platform: 'linux', uid: 1000 }; + assert.equal(launcher.ownedDir(dir, { ...opts, lstat: st(1000, 0o40700) }), dir); + for (const [lstat, why] of [[st(2000, 0o40700), /belongs to another user/], [st(1000, 0o40775), /writable by group or others/], [st(1000, 0o40707), /writable by group or others/]]) { + assert.throws(() => launcher.ownedDir(dir, { ...opts, lstat }), (err) => { + assert.match(err.message, why); + assert.match(err.hint, /^Remove .* and run dg again/); + assert.doesNotMatch(err.hint, /chmod/); + return true; + }); + } + // Windows has no uid or mode bits to go on; only links are refused there. + assert.equal(launcher.ownedDir(dir, { platform: 'win32', lstat: st(2000, 0o40777) }), dir); + }); + + test('a uv cache others can write to under a trusted root: a one-run venv instead, nothing used from it', posixOnly, async () => { + const home = tmpdir(); + const cache = path.join(home, 'c'); + const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); + fs.writeFileSync(requirementsFile, ''); + const uvCache = path.join(cache, 'deepgram-cli', 'uv-cache'); + fs.mkdirSync(uvCache, { recursive: true }); + fs.chmodSync(uvCache, 0o777); + const calls = []; + const lines = []; + const spawn = fakeSpawn((command, args) => { + calls.push(args); + if (args[0] === 'venv') fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + if (args[0] === 'pip') for (const f of ['python', 'deepctl']) fs.writeFileSync(path.join(path.dirname(args[args.indexOf('--python') + 1]), f), ''); + return 0; + }); + const r = await launcher.resolveRunner([], { + env: { PATH: '/bin', XDG_CACHE_HOME: cache }, + version: '1.2.3', + platform: 'linux', + home, + isExecutable: (f) => f === '/bin/uv', + requirementsFile, + prepareRoot: (spec) => fs.mkdirSync(spec.dir, { recursive: true }) || true, + spawn, + stderr: { write: (t) => lines.push(t) }, + }); + assert.ok(r.cleanup, 'a one-run venv'); + for (const c of calls) assert.equal(c[c.indexOf('--cache-dir') + 1], path.join(r.cleanup, '.setup-cache')); + assert.match(lines.join(''), new RegExp(`${uvCache.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} isn't one\\.\\nHINT: Remove `)); + }); + + test('a UV_CACHE_DIR the user set is left to uv', async () => { + const home = tmpdir(); + const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); + fs.writeFileSync(requirementsFile, ''); + const calls = []; + const spawn = fakeSpawn((command, args) => { + calls.push(args); + if (args[0] === 'venv') fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + if (args[0] === 'pip') for (const f of ['python', 'deepctl']) fs.writeFileSync(path.join(path.dirname(args[args.indexOf('--python') + 1]), f), ''); + return 0; + }); + await launcher.resolveRunner([], { + env: { PATH: '/bin', XDG_CACHE_HOME: path.join(home, 'c'), UV_CACHE_DIR: '/mine' }, + version: '1.2.3', + platform: 'linux', + home, + realpath: (f) => f, + isExecutable: (f) => f === '/bin/uv', + requirementsFile, + prepareRoot: (spec) => fs.mkdirSync(spec.dir, { recursive: true }) || true, + trustRoot: () => false, + spawn, + stderr: { write: () => {} }, + }); + assert.equal(calls.length, 2); + for (const c of calls) assert.equal(c.includes('--cache-dir'), false); + }); + test('a missing requirements file is refused before anything is installed', async () => { let spawned = false; await assert.rejects( @@ -1091,7 +1304,7 @@ describe('bin/dg.js end to end', posixOnly, () => { assert.equal(r.stdout, 'npm listen a b.wav\n'); const calls = readLog(log); assert.equal(calls.length, 2); - assert.match(calls[1], new RegExp(`^pip install --no-config --quiet --python \\S+/bin/python --require-hashes --no-deps -r ${pkg.requirementsFile} --only-binary :all:$`)); + assert.match(calls[1], new RegExp(`^pip install --no-config --quiet --cache-dir \\S+/deepgram-cli/uv-cache --python \\S+/bin/python --require-hashes --no-deps -r ${pkg.requirementsFile} --only-binary :all:$`)); // The second run reuses the venv: uv is not called again. assert.equal(runCopy(pkg, env, ['x']).stdout, 'npm x\n'); assert.equal(readLog(log).length, 2); From 2993b03fb79611718e125fb8503dc21a1baca263 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:13:25 -0400 Subject: [PATCH 09/15] refactor(core): keep the launcher module internal deepctl_core.launcher becomes deepctl_core._launcher, with no public re-export, so plugins don't build on it. deepctl-cmd-update, deepctl-cmd-skills, deepctl.main and the release's post-publish import check use the new name. capture_launcher() now takes the value from deepctl.main when that module has already captured DEEPCTL_LAUNCHER at startup (LAUNCHER_CAPTURED), and reads and removes the variable itself otherwise, so an entry point that captures it first keeps working. --- .github/workflows/release.yml | 4 +-- conftest.py | 2 +- .../src/deepctl_cmd_skills/startup_check.py | 2 +- .../tests/unit/test_skills_command.py | 2 +- .../src/deepctl_cmd_update/installation.py | 2 +- .../src/deepctl_cmd_update/startup_check.py | 2 +- .../{launcher.py => _launcher.py} | 22 ++++++++++---- .../deepctl-core/tests/unit/test_launcher.py | 30 +++++++++++++++++-- scripts/check_dependency_floors.py | 4 +-- scripts/check_release_bumps.py | 2 +- src/deepctl/main.py | 2 +- tests/unit/test_dependency_floors.py | 2 +- 12 files changed, 56 insertions(+), 20 deletions(-) rename packages/deepctl-core/src/deepctl_core/{launcher.py => _launcher.py} (64%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fdeca8f..fcdec38d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -157,7 +157,7 @@ jobs: # `chore:` commit under a package moves its code without a bump, and # PyPI keeps serving the old code under the same version: a clean # install then pairs new siblings with an old deepctl-core that has no - # deepctl_core.launcher. Refuse to publish in that state. + # deepctl_core._launcher. Refuse to publish in that state. - name: Check every changed package is being released run: python scripts/check_release_bumps.py @@ -288,7 +288,7 @@ jobs: /tmp/real/bin/deepctl --version # What the npm launcher and `dg update` rely on: a sibling released # with stale code fails here with ModuleNotFoundError. - /tmp/real/bin/python -c "import deepctl_core.launcher, deepctl_cmd_update.installation" + /tmp/real/bin/python -c "import deepctl_core._launcher, deepctl_cmd_update.installation" env: TAG_NAME: ${{ needs.release-please.outputs.tag_name }} diff --git a/conftest.py b/conftest.py index 89d1f5c3..7721c5c8 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ def _clean_deepgram_env(monkeypatch): @pytest.fixture(autouse=True) def _fresh_launcher_capture(): """Re-read DEEPCTL_LAUNCHER in each test, so env patches take effect.""" - from deepctl_core.launcher import reset_launcher_for_tests + from deepctl_core._launcher import reset_launcher_for_tests reset_launcher_for_tests() yield diff --git a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py index ed6ae711..3ed41ff7 100644 --- a/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py +++ b/packages/deepctl-cmd-skills/src/deepctl_cmd_skills/startup_check.py @@ -11,7 +11,7 @@ import threading from pathlib import Path -from deepctl_core.launcher import launched_via_npm +from deepctl_core._launcher import launched_via_npm # Environment variables that indicate a CI environment _CI_ENV_VARS = ( diff --git a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py index 9717c0b2..f511ce30 100644 --- a/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py +++ b/packages/deepctl-cmd-skills/tests/unit/test_skills_command.py @@ -975,7 +975,7 @@ def test_npm_launcher_is_not_oneshot(self): } with patch.dict(os.environ, env, clear=False): assert startup_check._is_oneshot() is False - from deepctl_core.launcher import reset_launcher_for_tests + from deepctl_core._launcher import reset_launcher_for_tests reset_launcher_for_tests() with patch.dict(os.environ, env, clear=False): diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py index c75e864d..63ccd5d4 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/installation.py @@ -7,7 +7,7 @@ from enum import Enum from pathlib import Path -from deepctl_core.launcher import get_launcher, launched_via_npm +from deepctl_core._launcher import get_launcher, launched_via_npm from pydantic import BaseModel diff --git a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py index bbaec963..d5b9296d 100644 --- a/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py +++ b/packages/deepctl-cmd-update/src/deepctl_cmd_update/startup_check.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any -from deepctl_core.launcher import launched_via_npm, npm_upgrade_command +from deepctl_core._launcher import launched_via_npm, npm_upgrade_command # Sentinel used when the background check is suppressed or has no result _NO_RESULT: dict[str, Any] = {} diff --git a/packages/deepctl-core/src/deepctl_core/launcher.py b/packages/deepctl-core/src/deepctl_core/_launcher.py similarity index 64% rename from packages/deepctl-core/src/deepctl_core/launcher.py rename to packages/deepctl-core/src/deepctl_core/_launcher.py index 58b31d3c..98075b85 100644 --- a/packages/deepctl-core/src/deepctl_core/launcher.py +++ b/packages/deepctl-core/src/deepctl_core/_launcher.py @@ -1,8 +1,8 @@ -"""Which wrapper, if any, started deepctl. +"""Which wrapper, if any, started deepctl (internal; not a public API). -The ``@deepgram/cli`` npm package runs deepctl from a private venv or through -``uv tool run`` and says so in ``DEEPCTL_LAUNCHER``: ``npm`` for a global -install, ``npx`` for ``npx``/``npm exec``. Nothing else sets it. +The ``@deepgram/cli`` npm package runs deepctl from a private venv and says +so in ``DEEPCTL_LAUNCHER``: ``npm`` for a global install, ``npx`` for +``npx``/``npm exec``. Nothing else sets it. The variable is read once and removed from ``os.environ``, so it describes this process only. Anything deepctl spawns -- a starter app, a shell, another @@ -12,6 +12,7 @@ from __future__ import annotations import os +import sys LAUNCHER_ENV_VAR = "DEEPCTL_LAUNCHER" NPM_LAUNCHERS = frozenset({"npm", "npx"}) @@ -21,10 +22,19 @@ def capture_launcher() -> str | None: - """Read and remove ``DEEPCTL_LAUNCHER`` (first call only); return its value.""" + """The launcher that started deepctl, captured on the first call. + + If an entry point captured it first and says so (a ``deepctl.main`` + module with ``LAUNCHER_CAPTURED`` set), use that value; otherwise read + and remove ``DEEPCTL_LAUNCHER`` here. + """ global _captured, _launcher if not _captured: - _launcher = os.environ.pop(LAUNCHER_ENV_VAR, None) or None + entry = sys.modules.get("deepctl.main") + if entry is not None and getattr(entry, "LAUNCHER_CAPTURED", False): + _launcher = entry.LAUNCHER + else: + _launcher = os.environ.pop(LAUNCHER_ENV_VAR, None) or None _captured = True return _launcher diff --git a/packages/deepctl-core/tests/unit/test_launcher.py b/packages/deepctl-core/tests/unit/test_launcher.py index ae954b26..76a3f827 100644 --- a/packages/deepctl-core/tests/unit/test_launcher.py +++ b/packages/deepctl-core/tests/unit/test_launcher.py @@ -1,9 +1,9 @@ -"""Unit tests for deepctl_core.launcher.""" +"""Unit tests for deepctl_core._launcher.""" import os from unittest.mock import patch -from deepctl_core import launcher +from deepctl_core import _launcher as launcher @patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "npm"}, clear=False) @@ -43,3 +43,29 @@ def test_npx_upgrade_command(): @patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "something-else"}, clear=False) def test_unknown_launcher_is_not_npm(): assert launcher.launched_via_npm() is False + + +@patch.dict(os.environ, {"DEEPCTL_LAUNCHER": "npx"}, clear=False) +def test_uses_the_value_deepctl_main_captured_at_startup(): + """An entry point that captured it first wins; the environment is left alone.""" + import sys + import types + + entry = types.ModuleType("deepctl.main") + entry.LAUNCHER = "npm" + entry.LAUNCHER_CAPTURED = True + with patch.dict(sys.modules, {"deepctl.main": entry}): + assert launcher.get_launcher() == "npm" + assert os.environ["DEEPCTL_LAUNCHER"] == "npx" + + +def test_the_module_is_internal(): + """No public ``deepctl_core.launcher`` to build plugins against.""" + import importlib + + import deepctl_core + import pytest + + assert not hasattr(deepctl_core, "launcher") + with pytest.raises(ModuleNotFoundError): + importlib.import_module("deepctl_core.launcher") diff --git a/scripts/check_dependency_floors.py b/scripts/check_dependency_floors.py index 58926708..f1800ad3 100644 --- a/scripts/check_dependency_floors.py +++ b/scripts/check_dependency_floors.py @@ -29,7 +29,7 @@ 4. **Coupled sibling floors equal workspace versions.** Some sub-packages import a sibling API that only exists in the sibling's newest release - (deepctl-cmd-update needs `deepctl_core.launcher`; deepctl-cmd-plugin + (deepctl-cmd-update needs `deepctl_core._launcher`; deepctl-cmd-plugin needs `InstallMethod.NPM` from deepctl-cmd-update). Rule 2 only stops a floor from going *above* the sibling, so a stale floor lets pip pair the new package with an old published sibling that lacks the API, which is an @@ -86,7 +86,7 @@ # version (rule 4). Add a pair when a package starts importing an API that # only the sibling's newest release has; the floor then tracks every release. SIBLING_FLOORS: dict[str, tuple[str, ...]] = { - # deepctl_core.launcher (npm launcher detection) + # deepctl_core._launcher (npm launcher detection) "deepctl-cmd-update": ("deepctl-core",), "deepctl-cmd-skills": ("deepctl-core",), # deepctl_cmd_update.installation.InstallMethod.NPM diff --git a/scripts/check_release_bumps.py b/scripts/check_release_bumps.py index 6962a553..cc79ecfa 100644 --- a/scripts/check_release_bumps.py +++ b/scripts/check_release_bumps.py @@ -8,7 +8,7 @@ then keeps serving the old code under the same version number, and a sibling that needs the new code installs next to it and fails at import. On this branch, a clean `pip install` of the root wheel resolved -deepctl-core 0.3.0 from PyPI, which has no `deepctl_core.launcher`. +deepctl-core 0.3.0 from PyPI, which has no `deepctl_core._launcher`. So before anything is published: for every package in the release-please manifest whose current version already has a release tag, `src/` and the diff --git a/src/deepctl/main.py b/src/deepctl/main.py index 7b67a33f..085fe408 100644 --- a/src/deepctl/main.py +++ b/src/deepctl/main.py @@ -19,7 +19,7 @@ print_timing_summary, setup_output, ) -from deepctl_core.launcher import capture_launcher +from deepctl_core._launcher import capture_launcher from deepctl_core.output import stderr_console from rich.traceback import install diff --git a/tests/unit/test_dependency_floors.py b/tests/unit/test_dependency_floors.py index 6ade2457..b1d09c59 100644 --- a/tests/unit/test_dependency_floors.py +++ b/tests/unit/test_dependency_floors.py @@ -241,7 +241,7 @@ def test_final_release_floor_above_a_pre_release_is_unsatisfiable( # --------------------------------------------------------------------------- # Rule 4: coupled sibling floors (SIBLING_FLOORS) track the sibling's -# workspace version. deepctl-cmd-update imports deepctl_core.launcher, which +# workspace version. deepctl-cmd-update imports deepctl_core._launcher, which # no published deepctl-core had; with a stale floor pip can pair the new # update package with an old core and `dg` fails at import. # --------------------------------------------------------------------------- From ba145897daa9dfb5ad8f4e275a09dd0008b58e70 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:31:24 -0400 Subject: [PATCH 10/15] test(npm): keep launcher tests independent of the host filesystem Run the two real-filesystem tests for the Linux cache path only on Linux and macOS, as the other real-filesystem tests already are, and compare package paths against the realpath of the temp directory, which macOS reports under /private/var. Add a Windows test with a short-name (8.3) LOCALAPPDATA to show only link and junction flags decide whether a directory is safe. --- npm/lib/launcher.js | 5 ++++- npm/test/launcher.test.js | 19 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 9aa5b2f2..6beb67b1 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -554,7 +554,10 @@ function ownedDir(dir, opts = {}) { err.dir = dir; return err; }; - mkdirPrivate(dir); + if (opts.mkdir) opts.mkdir(dir, { mode: 0o700 }); + else mkdirPrivate(dir); + // lstat only: a link or junction is refused for what it is. Nothing here + // compares path spellings, so an 8.3 short name (RUNNER~1) is fine. const st = (opts.lstat || fs.lstatSync)(dir); if (st.isSymbolicLink() || !st.isDirectory()) throw refuse('is a link, not a plain directory'); if (!isWindows(opts.platform || process.platform)) { diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index 95b24534..c0ac4e50 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -54,7 +54,9 @@ const posixOnly = { skip: process.platform === 'win32' && 'POSIX shell scripts' // directory does not have (the release writes it). `requirements: null` // leaves the file out. function packageCopy(requirements = 'deepctl==0.0.0 \\\n --hash=sha256:0000\n') { - const dir = tmpdir(); + // Real path: Node resolves the launcher's __dirname, so on macOS the + // package lives under /private/var/..., not the /var/... tmpdir() gives. + const dir = fs.realpathSync(tmpdir()); for (const sub of ['bin', 'lib']) fs.cpSync(path.join(PKG_DIR, sub), path.join(dir, sub), { recursive: true }); fs.copyFileSync(path.join(PKG_DIR, 'package.json'), path.join(dir, 'package.json')); const version = launcher.packageVersion(PKG_DIR); @@ -773,6 +775,16 @@ describe('no trusted cache root: a one-run venv instead', () => { assert.throws(() => launcher.oneRunBase(undefined, { ...win, lstat: lstatWith(linked) }), /link or junction/); } assert.deepEqual(launcher.oneRunBase(undefined, { ...win, lstat: lstatWith(null) }), { base: `${top}\\venvs`, prefix: '', prune: true }); + // An 8.3 short name (C:\\Users\\RUNNER~1, as on GitHub's runners) is a + // plain directory like any other: only links and junctions are refused, + // and no short and long spellings are ever compared. + const short = { ...win, env: { LOCALAPPDATA: 'C:\\Users\\RUNNER~1\\AppData\\Local' }, home: 'C:\\Users\\RUNNER~1' }; + assert.deepEqual(launcher.oneRunBase(undefined, { ...short, lstat: lstatWith(null), realpath: () => 'C:\\Users\\runneradmin\\AppData\\Local' }), { + base: 'C:\\Users\\RUNNER~1\\AppData\\Local\\deepgram-cli\\venvs', + prefix: '', + prune: true, + }); + assert.equal(launcher.ownedDir('C:\\Users\\RUNNER~1\\AppData\\Local\\deepgram-cli\\uv-cache', { platform: 'win32', lstat: lstatWith(null), mkdir: () => {} }), 'C:\\Users\\RUNNER~1\\AppData\\Local\\deepgram-cli\\uv-cache'); let spawned = false; await assert.rejects( launcher.resolveRunner([], { @@ -1064,7 +1076,8 @@ describe('resolveRunner', () => { await assert.rejects(launcher.resolveRunner([], { env: { DEEPCTL_NPM_RUNNER: 'pipx' }, version: '1.0.0' }), /must be one of/); }); - test('with no venv cached, uv builds one from the hash-locked requirements file', async () => { + // POSIX semantics on the real filesystem (platform: 'linux'). + test('with no venv cached, uv builds one from the hash-locked requirements file', posixOnly, async () => { const home = tmpdir(); const cache = path.join(home, 'c'); const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); @@ -1158,7 +1171,7 @@ describe('resolveRunner', () => { assert.match(lines.join(''), new RegExp(`${uvCache.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} isn't one\\.\\nHINT: Remove `)); }); - test('a UV_CACHE_DIR the user set is left to uv', async () => { + test('a UV_CACHE_DIR the user set is left to uv', posixOnly, async () => { const home = tmpdir(); const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); fs.writeFileSync(requirementsFile, ''); From 7191c5e1793fcce0ddc2c639db0c7fb3124c34c8 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Sat, 31 Oct 2026 12:37:26 -0400 Subject: [PATCH 11/15] test(npm): shadow every Python the launcher probes in the fake-Python tests The launcher now tries python3.14 first, but the end-to-end and signal tests only installed fake python3.13 to python3 executables. On a machine with a real python3.14 on PATH, such as GitHub's ubuntu runners, the launcher found that one instead, built a real venv and ran a real pip, so those tests failed or timed out waiting for the fake step. The fakes now cover every name the launcher probes, from one list. Regular-expression escaping in the tests also goes through one helper that escapes backslashes and every other special character. --- npm/test/launcher.test.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index c0ac4e50..1ef593e4 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -49,6 +49,15 @@ function fakeSpawn(body) { const posixOnly = { skip: process.platform === 'win32' && 'POSIX shell scripts' }; +// Every name pythonCandidates probes on POSIX. A fake Python must shadow +// all of them, or a real interpreter later on PATH (GitHub's ubuntu +// runners have several) wins and builds a real venv. +const PYTHON_NAMES = ['python3.14', 'python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']; + +function escapeRegExp(text) { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + // A copy of the package as a release tarball has it: bin/, lib/, // package.json and requirements-.txt, which the repository's npm/ // directory does not have (the release writes it). `requirements: null` @@ -718,7 +727,7 @@ describe('no trusted cache root: a one-run venv instead', () => { for (const c of calls) assert.equal(c[c.indexOf('--cache-dir') + 1], path.join(r.cleanup, '.setup-cache')); assert.equal(fs.existsSync(path.join(r.cleanup, '.setup-cache')), false); for (const spec of specs) assert.deepEqual(fs.readdirSync(spec.dir), ['1.0.0'], 'nothing written into an untrusted root'); - assert.match(stderr, new RegExp(`for this run only: .*${specs[0].dir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} isn't one\\.\\nHINT: `)); + assert.match(stderr, new RegExp(`for this run only: .*${escapeRegExp(specs[0].dir)} isn't one\\.\\nHINT: `)); } test('group-writable roots are not reused or written to', posixOnly, async () => { @@ -1168,7 +1177,7 @@ describe('resolveRunner', () => { }); assert.ok(r.cleanup, 'a one-run venv'); for (const c of calls) assert.equal(c[c.indexOf('--cache-dir') + 1], path.join(r.cleanup, '.setup-cache')); - assert.match(lines.join(''), new RegExp(`${uvCache.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} isn't one\\.\\nHINT: Remove `)); + assert.match(lines.join(''), new RegExp(`${escapeRegExp(uvCache)} isn't one\\.\\nHINT: Remove `)); }); test('a UV_CACHE_DIR the user set is left to uv', posixOnly, async () => { @@ -1348,7 +1357,7 @@ describe('bin/dg.js end to end', posixOnly, () => { assert.equal(r.status, 1); assert.equal(r.stdout, ''); assert.match(r.stderr, /Hash mismatch/); - assert.match(r.stderr, new RegExp(`ERROR: Installing deepctl==${pkg.version.replace(/\./g, '\\.')} failed \\(exit 1\\)\\.\\nHINT: .*hash mismatch above means a download did not match this release`)); + assert.match(r.stderr, new RegExp(`ERROR: Installing deepctl==${escapeRegExp(pkg.version)} failed \\(exit 1\\)\\.\\nHINT: .*hash mismatch above means a download did not match this release`)); assert.equal(fs.existsSync(ran), false); assert.deepEqual(fs.readdirSync(launcher.cacheRoots({ env, home })[0]), []); }); @@ -1404,7 +1413,7 @@ describe('bin/dg.js end to end', posixOnly, () => { 'exit 0', '', ].join('\n'); - for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) { + for (const name of PYTHON_NAMES) { fakeExe(bin, name, fakePython); } fs.writeFileSync(path.join(home, 'checker'), checker); @@ -1447,7 +1456,7 @@ describe('bin/dg.js end to end', posixOnly, () => { const home = tmpdir(); const bin = path.join(home, 'bin'); const fakePython = '#!/bin/sh\n[ "$1" = -I ] && shift\ncase "$1 $2" in\n "-c "*) echo "(3, 12)"; exit 0;;\nesac\nkill -KILL $$\n'; - for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) fakeExe(bin, name, fakePython); + for (const name of PYTHON_NAMES) fakeExe(bin, name, fakePython); const r = runCopy(packageCopy(), { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: path.join(home, 'c'), DEEPCTL_NPM_RUNNER: 'venv' }); assert.equal(r.status, 1); assert.match(r.stderr, /failed \(killed by SIGKILL\)/); @@ -1513,7 +1522,7 @@ describe('a signal during first-run setup', posixOnly, () => { 'exit 0', '', ].join('\n'); - for (const name of ['python3.13', 'python3.12', 'python3.11', 'python3.10', 'python3']) fakeExe(bin, name, fakePython); + for (const name of PYTHON_NAMES) fakeExe(bin, name, fakePython); const cache = path.join(home, 'c'); const env = { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: cache, DEEPCTL_NPM_RUNNER: 'venv', HANG_AT: hangAt }; // Where the launcher itself will put venvs and locks: XDG_CACHE_HOME From fac168ebe00587b177167a94f7959a6fb4fece20 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Mon, 2 Nov 2026 21:23:25 -0500 Subject: [PATCH 12/15] fix(npm): never trust a group-writable cache ancestor A group id equal to our uid (the user-private-group convention) does not prove the group has no other members: an administrator can add one, and Node cannot list them reliably. Any ancestor writable by group or others now has to be sticky, like /tmp. A umask-002 ~/.cache (0775) therefore fails, and the venvs go to the $TMPDIR/deepgram-cli- root, which is held to the same checks and still caches across runs. The npm README says where the cache goes in that case. Co-Authored-By: Claude Opus 5.5 --- npm/README.md | 10 ++++-- npm/lib/launcher.js | 24 ++++++--------- npm/test/launcher.test.js | 64 ++++++++++++++++++++++++++------------- 3 files changed, 60 insertions(+), 38 deletions(-) diff --git a/npm/README.md b/npm/README.md index 71ebd912..1a8d410a 100644 --- a/npm/README.md +++ b/npm/README.md @@ -58,9 +58,13 @@ working: `UV_CACHE_DIR`. The launcher reuses that cache only when you own it, nobody else can write to -it, and it isn't a symlink. If no cache directory qualifies, each run builds a -fresh environment in a private temporary directory and deletes it afterwards, -and says why on stderr. +it, and it isn't a symlink. The folders above it count too: on Linux, if +your group can write to `~/.cache` (or `$XDG_CACHE_HOME`; umask 002 leaves it +`0775`), the launcher keeps its cache in `$TMPDIR/deepgram-cli-` +instead, until your temp directory is cleaned. `chmod g-w ~/.cache` brings it +back to `~/.cache`. If no cache directory qualifies, each run builds a fresh +environment in a private temporary directory and deletes it afterwards, and +says why on stderr. On Windows, step 1 is skipped: the launcher can't verify who controls a cache directory there, so it never runs an environment left by an earlier run. Every diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 6beb67b1..676671d7 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -515,17 +515,16 @@ function rootTrusted(spec, opts = {}) { } catch { return false; } - const gid = opts.gid !== undefined ? opts.gid : process.getgid ? process.getgid() : null; - return ancestorsSafe(above, uid, gid, lstat, true) && ancestorsSafe(real, uid, gid, lstat, false); + return ancestorsSafe(above, uid, lstat, true) && ancestorsSafe(real, uid, lstat, false); } -// Group-writable is trusted only for a directory we own whose group id -// equals our uid: the user-private-group convention (Ubuntu, Fedora; umask -// 002), where ~/.cache is 0775 with a group named after us. A shared -// primary group (SUSE and Debian USERGROUPS=no use `users`, macOS uses -// `staff`) has other members who could swap the cache during the first -// build, so anything else group-writable fails closed. -function ancestorsSafe(start, uid, gid, lstat, allowLinks) { +// Group write counts as writable by others, even on a directory we own +// whose group id equals our uid (the user-private-group convention, where +// umask 002 leaves ~/.cache 0775): nothing guarantees that group has no +// other members, and Node cannot list them reliably (primary-group members +// and NSS sources such as LDAP are not in /etc/group). Such a ~/.cache +// fails, and the venvs go to the $TMPDIR root instead (cacheRootSpecs). +function ancestorsSafe(start, uid, lstat, allowLinks) { for (let dir = start; ; dir = path.posix.dirname(dir)) { let st; try { @@ -536,9 +535,7 @@ function ancestorsSafe(start, uid, gid, lstat, allowLinks) { const link = st.isSymbolicLink ? st.isSymbolicLink() : false; if (link ? !allowLinks : !st.isDirectory()) return false; if (st.uid !== 0 && st.uid !== uid) return false; - const privateGroup = st.uid === uid && st.gid === gid && gid === uid; - const writableByOthers = (st.mode & 0o002) !== 0 || ((st.mode & 0o020) !== 0 && !privateGroup); - if (!link && writableByOthers && (st.mode & STICKY) === 0) return false; + if (!link && (st.mode & 0o022) !== 0 && (st.mode & STICKY) === 0) return false; if (dir === '/' || dir === '.') return true; } } @@ -949,8 +946,7 @@ function oneRunBase(untrusted, opts = {}) { if (untrusted) { const tmp = (opts.realpath || fs.realpathSync)(opts.tmpdir || os.tmpdir()); const uid = currentUid(opts); - const gid = opts.gid !== undefined ? opts.gid : process.getgid ? process.getgid() : null; - if (!ancestorsSafe(tmp, uid, gid, lstat, false)) throw new Error(`${tmp} is not safe from other users`); + if (!ancestorsSafe(tmp, uid, lstat, false)) throw new Error(`${tmp} is not safe from other users`); return { base: tmp, prefix: 'deepgram-cli-', prune: false }; } const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index 1ef593e4..31a1be02 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -551,9 +551,9 @@ describe('cache root ownership (POSIX)', () => { '/home/u': dirStat(1000, 0o755), '/home/u/.cache': dirStat(1000, 0o700), }; - const check = (spec, stats, realpath = (d) => d, gid = 1000) => { + const check = (spec, stats, realpath = (d) => d) => { const all = { ...ancestors, ...stats }; - return launcher.rootTrusted(spec, { platform: 'linux', uid: 1000, gid, realpath, lstat: (d) => { + return launcher.rootTrusted(spec, { platform: 'linux', uid: 1000, realpath, lstat: (d) => { if (!(d in all)) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); return all[d]; } }); @@ -628,27 +628,20 @@ describe('cache root ownership (POSIX)', () => { assert.equal(launcher.rootTrusted(link), false); }); - test('our own group-writable ~/.cache in our private group is accepted (umask 002)', () => { + test('a group-writable ancestor is rejected even when it is ours and its gid equals our uid', () => { + // The user-private-group convention (umask 002) proves nothing: an + // administrator can add another account to group 1000, which could then + // swap the cache between our check and our exec. const base = '/home/u/.cache/deepgram-cli'; const ok = { [base]: dirStat(1000, 0o700), [`${base}/venvs`]: dirStat(1000, 0o700) }; - assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775) }), true); - assert.equal(check(home, { ...ok, '/home/u': dirStat(1000, 0o775) }), true); - // Group-writable with a shared group (staff, users): rejected. - assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 50) }), false); + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 1000) }), false); + assert.equal(check(home, { ...ok, '/home/u': dirStat(1000, 0o775, true, 1000) }), false); + // A shared primary group (staff, users): rejected too. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 100) }), false); // Writable by everyone: rejected even when ours. assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o777) }), false); - // Group-writable, our gid, but owned by someone else: rejected. - assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(2000, 0o775, true, 1000) }), false); - }); - - test('a group-writable ~/.cache in a shared primary group is rejected even though it is ours', () => { - // SUSE / Debian USERGROUPS=no: primary group `users` (100), shared with - // every other user, so gid matches ours but is not a private group. - const base = '/home/u/.cache/deepgram-cli'; - const ok = { [base]: dirStat(1000, 0o700, true, 100), [`${base}/venvs`]: dirStat(1000, 0o700, true, 100) }; - assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o775, true, 100) }, (d) => d, 100), false); - // The same directory without group write is fine. - assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o755, true, 100) }, (d) => d, 100), true); + // The same directory without group write is fine, whatever its group. + assert.equal(check(home, { ...ok, '/home/u/.cache': dirStat(1000, 0o755, true, 100) }), true); }); test('prepareRoot creates missing parents 0700 even under umask 002', posixOnly, () => { @@ -752,7 +745,7 @@ describe('no trusted cache root: a one-run venv instead', () => { test('roots owned by another user are not reused (lstat stand-in)', async () => { const st = (uid, mode = 0o40700) => ({ uid, gid: uid, mode, isDirectory: () => true, isSymbolicLink: () => false }); - const opts = { platform: 'linux', uid: 1000, gid: 1000, realpath: (f) => f }; + const opts = { platform: 'linux', uid: 1000, realpath: (f) => f }; for (const shared of [false, true]) { const spec = { dir: '/h/.cache/deepgram-cli/venvs', shared }; assert.equal(launcher.rootTrusted(spec, { ...opts, lstat: (d) => (d.startsWith('/h/.cache/deepgram-cli') ? st(2000) : st(0, 0o40755)) }), false); @@ -761,7 +754,7 @@ describe('no trusted cache root: a one-run venv instead', () => { test('the temp directory must itself be safe from other users', () => { const st = (uid, mode) => ({ uid, gid: uid, mode, isDirectory: () => true, isSymbolicLink: () => false }); - const base = { platform: 'linux', uid: 1000, gid: 1000, realpath: (f) => f, tmpdir: '/scratch' }; + const base = { platform: 'linux', uid: 1000, realpath: (f) => f, tmpdir: '/scratch' }; // World-writable without the sticky bit: anyone could rename our directory. assert.throws(() => launcher.oneRunBase('/h/.cache', { ...base, lstat: (d) => (d === '/scratch' ? st(0, 0o40777) : st(0, 0o40755)) }), /not safe/); // /tmp-style sticky 1777 is fine. @@ -1332,6 +1325,35 @@ describe('bin/dg.js end to end', posixOnly, () => { assert.equal(readLog(log).length, 2); }); + test('a group-writable ~/.cache (umask 002) caches in the temp root instead, and reuses it', () => { + const home = tmpdir(); + const bin = path.join(home, 'bin'); + const log = path.join(home, 'uv.log'); + fakeUv(bin, '#!/bin/sh\necho "ran $*"\n', log); + // macOS caches under ~/Library/Caches and ignores XDG_CACHE_HOME. + const cache = process.platform === 'darwin' ? path.join(home, 'Library', 'Caches') : path.join(home, 'c'); + const tmp = path.join(home, 't'); + fs.mkdirSync(cache, { recursive: true }); + fs.chmodSync(cache, 0o775); + fs.mkdirSync(tmp, { mode: 0o700 }); + const pkg = packageCopy(); + const env = { PATH: `${bin}:/usr/bin:/bin`, HOME: home, XDG_CACHE_HOME: cache, TMPDIR: tmp }; + const r = runCopy(pkg, env, ['a']); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, 'ran a\n'); + assert.doesNotMatch(r.stderr, /for this run only/); + const calls = readLog(log); + assert.equal(calls.length, 2); + const root = path.join(tmp, `deepgram-cli-${process.getuid()}`); + assert.ok(calls[1].includes(`--cache-dir ${path.join(root, 'uv-cache')} `), calls[1]); + assert.equal(fs.readdirSync(path.join(root, 'venvs')).filter((e) => !e.endsWith('.lock')).length, 1); + // Nothing was built under the group-writable ~/.cache. + assert.deepEqual(fs.readdirSync(path.join(cache, 'deepgram-cli', 'venvs')), []); + // The second run reuses the temp root's venv: uv is not called again. + assert.equal(runCopy(pkg, env, ['b']).stdout, 'ran b\n'); + assert.equal(readLog(log).length, 2); + }); + test('without its requirements file the package installs nothing and says how to fix it', () => { const home = tmpdir(); const bin = path.join(home, 'bin'); From a9bb96767454e307720e537767b7bdc492c50a2c Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Mon, 2 Nov 2026 21:23:25 -0500 Subject: [PATCH 13/15] fix(npm): never reuse a uv cache the launcher can't check uv installs files it extracted into its cache without hashing them again, so a cache that outlives the run has to be as trusted as a venv. Windows cannot check owners, ACLs or reparse points from Node, so it no longer keeps %LOCALAPPDATA%\deepgram-cli\uv-cache: uv gets a cache inside the run's own directory, removed once setup is done. pip keeps its own cache, since with --require-hashes it hashes every file it installs. A UV_CACHE_DIR in the environment no longer switches the launcher's cache off: tools set it without the user choosing to (astral-sh/setup-uv exports it on GitHub's runners), so the launcher always passes --cache-dir, which uv puts ahead of UV_CACHE_DIR. The hash-locked Windows CI job plants a uv cache where the launcher used to keep it and fails if uv writes to it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 9 +++++++ npm/README.md | 18 ++++++++------ npm/lib/launcher.js | 23 +++++++++-------- npm/test/launcher.test.js | 51 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 82 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb9f3167..0117c9b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -281,6 +281,13 @@ jobs: python npm/scripts/write_requirements.py --index-url http://127.0.0.1:8765 --dist dist VERSION="$(node -p 'require("./npm/package.json").version')" export UV_INDEX_URL=http://127.0.0.1:8765/simple PIP_INDEX_URL=http://127.0.0.1:8765/simple + # Windows never reuses a uv cache: plant one where earlier versions + # kept it, and check below that uv neither read nor wrote it. + if [ "${RUNNER_OS}" = "Windows" ]; then + PLANTED="$(cygpath "${LOCALAPPDATA}")/deepgram-cli/uv-cache" + mkdir -p "${PLANTED}" + echo planted > "${PLANTED}/marker" + fi for runner in uvx venv; do # A fresh HOME per runner, so the venv run cannot reuse the venv # the uv run cached (macOS keeps it under ~/Library/Caches). @@ -294,6 +301,8 @@ jobs: if [ "${RUNNER_OS}" = "Windows" ]; then LEFT="$(ls -A "$(cygpath "${LOCALAPPDATA}")/deepgram-cli/venvs")" [ -z "${LEFT}" ] || { echo "::error::one-run venv left behind: ${LEFT}"; exit 1; } + CACHED="$(ls -A "${PLANTED}")" + [ "${CACHED}" = "marker" ] || { echo "::error::the uv cache in ${PLANTED} was used: ${CACHED}"; exit 1; } fi # The release PR's bump gate (scripts/check_release_bumps.py), for commits diff --git a/npm/README.md b/npm/README.md index 1a8d410a..7f021852 100644 --- a/npm/README.md +++ b/npm/README.md @@ -52,10 +52,12 @@ working: `UV_CONFIG_FILE`, and one `uv.toml`: your user file (`~/.config/uv/uv.toml`, or `%APPDATA%\uv\uv.toml` on Windows) if it exists, otherwise the system file (`/etc/uv/uv.toml`, or - `%PROGRAMDATA%\uv\uv.toml`). uv's `cache-dir` setting is the exception: - the launcher gives uv a cache of its own next to its environments, since uv - installs cached files without re-checking them, unless you set - `UV_CACHE_DIR`. + `%PROGRAMDATA%\uv\uv.toml`). uv's cache settings (`cache-dir` in + `uv.toml` and `UV_CACHE_DIR`) are the exception: the launcher always + gives uv a cache of its own, since uv installs cached files without + re-checking them. It lives next to its environments, or, when the + launcher builds an environment for one run only (always on Windows), + inside that run's directory and is deleted afterwards. The launcher reuses that cache only when you own it, nobody else can write to it, and it isn't a symlink. The folders above it count too: on Linux, if @@ -69,9 +71,11 @@ says why on stderr. On Windows, step 1 is skipped: the launcher can't verify who controls a cache directory there, so it never runs an environment left by an earlier run. Every run builds a fresh one under `%LOCALAPPDATA%\deepgram-cli` and deletes it -afterwards. With uv that takes a few seconds; without it, about a minute each -time, so install uv. The launcher needs a real `uv.exe` or `python.exe` on -`PATH`: pyenv-win's `python.bat` shims and `uv.cmd` wrappers are skipped. +afterwards, along with uv's cache, so each run downloads the packages again. +With uv that takes a few seconds plus the download; without it, about a +minute each time, so install uv. The launcher needs a real `uv.exe` or +`python.exe` on `PATH`: pyenv-win's `python.bat` shims and `uv.cmd` wrappers +are skipped. Launcher messages go to stderr, so stdout carries only the CLI's own output (`dg ... -o json`, `dg mcp`). diff --git a/npm/lib/launcher.js b/npm/lib/launcher.js index 676671d7..e1936005 100644 --- a/npm/lib/launcher.js +++ b/npm/lib/launcher.js @@ -966,14 +966,10 @@ async function buildPrivateVenv({ installer, key, spec, locked, version, untrust const stderr = opts.stderr || fdStderr; const p = isWindows(opts.platform || process.platform) ? path.win32 : path.posix; let dir; - let uvCache; try { const { base, prefix, prune } = oneRunBase(untrusted, opts); if (prune) pruneOneRunVenvs(base, p); dir = fs.mkdtempSync(p.join(base, `${prefix}${key}-`)); - // Windows: a uv cache kept beside the one-run venvs, held to the same - // plain-directory check (oneRunBase), so runs after the first are quick. - if (!untrusted && installer.uv) uvCache = ownedDir(p.join(p.dirname(base), 'uv-cache'), opts); } catch (err) { throw new LauncherError(`No safe, writable directory to set up the Deepgram CLI in: ${err.message}`, untrusted ? 'Point TMPDIR at a directory only you can write to.' : 'Point LOCALAPPDATA at a directory only you can write to.'); } @@ -984,11 +980,16 @@ async function buildPrivateVenv({ installer, key, spec, locked, version, untrust stderr.write(hint ? `HINT: ${hint}\n` : `HINT: Make it yours and not group- or world-writable (chmod go-w)${elsewhere}.\n`); } else if (installer.uv) stderr.write(`Setting up Deepgram CLI ${version} for this run...\n`); else stderr.write(`Setting up Deepgram CLI ${version} for this run (~1 min each time; install uv to speed it up: ${UV_INSTALL_WINDOWS})...\n`); - // No trusted root: uv's cache for this run lives inside the run's own - // directory and goes once setup is done; pip runs without a cache. - const privateCache = untrusted ? p.join(dir, '.setup-cache') : undefined; + // uv's cache for this run lives inside the run's own directory and goes + // once setup is done: uv reuses what it extracted into a cache without + // hashing it again, so a cache that outlives the run would have to be + // checked like a venv, and on Windows it cannot be (see cacheReuse). Each + // run downloads its wheels again. With no trusted root, pip runs without a + // cache too; otherwise pip keeps its own, which is safe because with + // --require-hashes pip hashes every file it installs, cached or not. + const privateCache = installer.uv ? p.join(dir, '.setup-cache') : undefined; try { - await installInto(installer, layout, spec, locked, opts, { cwd: dir, uvCache: privateCache || uvCache, noPipCache: Boolean(untrusted) }); + await installInto(installer, layout, spec, locked, opts, { cwd: dir, uvCache: privateCache, noPipCache: Boolean(untrusted) }); if (privateCache) fs.rmSync(privateCache, { recursive: true, force: true }); } catch (err) { fs.rmSync(dir, { recursive: true, force: true }); @@ -1075,8 +1076,10 @@ async function installInto(installer, layout, spec, locked, opts, where = {}) { // uv installs the files it extracted into its cache without hashing them // again, so its cache must be as private as the venv: never uv's default // ($XDG_CACHE_HOME/uv may be group-writable), always one the launcher - // owns, unless the user chose one with UV_CACHE_DIR. - const cache = where.uvCache && !env.UV_CACHE_DIR ? ['--cache-dir', where.uvCache] : []; + // owns. UV_CACHE_DIR is ignored for setup (--cache-dir wins over it): a + // cache the launcher didn't create can't be checked, and tools such as + // astral-sh/setup-uv set it without the user choosing to. + const cache = where.uvCache ? ['--cache-dir', where.uvCache] : []; const config = uvConfigArgs(env, opts); const uv = (sub) => [...sub, '--no-config', ...config, '--quiet', ...cache]; await runStep(installer.uv, [...uv(['venv']), '--allow-existing', '--python', PYTHON_RANGE, layout.dir], 'Creating the Python venv', UV_VENV_HINT, opts, where.cwd); diff --git a/npm/test/launcher.test.js b/npm/test/launcher.test.js index 31a1be02..8e165684 100644 --- a/npm/test/launcher.test.js +++ b/npm/test/launcher.test.js @@ -470,6 +470,47 @@ describe('Windows never reuses a cached venv', () => { assert.equal(fs.existsSync(path.join(r.cleanup, '.complete')), false); assert.equal(fs.existsSync(leaked), false, 'a leaked one-run venv was pruned'); }); + + for (const viaEnv of [false, true]) test(`with uv, a pre-existing uv cache is never used${viaEnv ? ', even when UV_CACHE_DIR names it' : ''}: uv gets a cache for this run only`, posixOnly, async () => { + // As above: reuse off stands in for win32. A uv cache left beside the + // venvs (by an earlier version, or by someone else) must not be read, + // and neither must one UV_CACHE_DIR points at (setup-uv sets it in CI). + const home = tmpdir(); + const cache = path.join(home, 'c'); + const planted = path.join(cache, 'deepgram-cli', 'uv-cache'); + fs.mkdirSync(planted, { recursive: true }); + fs.writeFileSync(path.join(planted, 'marker'), 'planted'); + const requirementsFile = path.join(home, 'requirements-1.0.0.txt'); + fs.writeFileSync(requirementsFile, ''); + const calls = []; + const spawn = fakeSpawn((command, args) => { + calls.push(args); + const cacheDir = args[args.indexOf('--cache-dir') + 1]; + fs.mkdirSync(cacheDir, { recursive: true }); + fs.writeFileSync(path.join(cacheDir, 'wheel'), ''); + if (args[0] === 'venv') fs.mkdirSync(path.join(args[args.length - 1], 'bin'), { recursive: true }); + if (args[0] === 'pip') for (const f of ['python', 'deepctl']) fs.writeFileSync(path.join(path.dirname(args[args.indexOf('--python') + 1]), f), ''); + return 0; + }); + const r = await launcher.resolveRunner(['x'], { + requirementsFile, + env: { PATH: '/bin', XDG_CACHE_HOME: cache, ...(viaEnv ? { UV_CACHE_DIR: planted } : {}) }, + version: '1.0.0', + platform: 'linux', + home, + cacheReuse: false, + isExecutable: (f) => f === '/bin/uv', + realpath: (f) => f, + spawn, + stderr: { write: () => {} }, + }); + assert.ok(r.cleanup, 'a one-run venv'); + assert.equal(calls.length, 2); + for (const c of calls) assert.equal(c[c.indexOf('--cache-dir') + 1], path.join(r.cleanup, '.setup-cache')); + // The run's cache is gone once setup is done; the planted one is untouched. + assert.equal(fs.existsSync(path.join(r.cleanup, '.setup-cache')), false); + assert.deepEqual(fs.readdirSync(planted), ['marker']); + }); }); describe("uv's own configuration", () => { @@ -1173,7 +1214,9 @@ describe('resolveRunner', () => { assert.match(lines.join(''), new RegExp(`${escapeRegExp(uvCache)} isn't one\\.\\nHINT: Remove `)); }); - test('a UV_CACHE_DIR the user set is left to uv', posixOnly, async () => { + test("a UV_CACHE_DIR in the environment is overridden by the launcher's own cache", posixOnly, async () => { + // astral-sh/setup-uv and similar tools export UV_CACHE_DIR without the + // user choosing it; a cache the launcher didn't create can't be checked. const home = tmpdir(); const requirementsFile = path.join(home, 'requirements-1.2.3.txt'); fs.writeFileSync(requirementsFile, ''); @@ -1198,7 +1241,11 @@ describe('resolveRunner', () => { stderr: { write: () => {} }, }); assert.equal(calls.length, 2); - for (const c of calls) assert.equal(c.includes('--cache-dir'), false); + const own = path.join(home, 'c', 'deepgram-cli', 'uv-cache'); + for (const c of calls) { + assert.equal(c[c.indexOf('--cache-dir') + 1], own); + assert.ok(!c.includes('/mine')); + } }); test('a missing requirements file is refused before anything is installed', async () => { From ea0f37f84c067f2bc063a8fe446b7a213a0eb415 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Mon, 2 Nov 2026 21:23:25 -0500 Subject: [PATCH 14/15] ci(release): gate the npm package and wheel metadata on a root bump The release bump gate watched only each package's src/. The root package also publishes @deepgram/cli, so a chore: or refactor: to npm/lib/ or npm/bin/ passed without a release. The root now also watches every entry in npm/package.json's `files`, read from HEAD so new entries are covered, with a glob matching at any depth as npm does, and package.json itself. npm/README.md and npm/LICENSE stay out, like the packages' READMEs: a docs change should not force a release. pyproject.toml is now compared on the metadata that reaches the wheel, not just dependencies: the [project] table without its version, [build-system], [tool.setuptools], and [tool.hatch]'s build and metadata tables. An entry point or requires-python change without a bump is caught too. Co-Authored-By: Claude Opus 5.5 --- scripts/check_release_bumps.py | 152 +++++++++++++--- tests/unit/test_release_bumps.py | 292 +++++++++++++++++++++++++++++-- 2 files changed, 408 insertions(+), 36 deletions(-) diff --git a/scripts/check_release_bumps.py b/scripts/check_release_bumps.py index cc79ecfa..94d4308d 100644 --- a/scripts/check_release_bumps.py +++ b/scripts/check_release_bumps.py @@ -11,16 +11,28 @@ deepctl-core 0.3.0 from PyPI, which has no `deepctl_core._launcher`. So before anything is published: for every package in the release-please -manifest whose current version already has a release tag, `src/` and the -package's dependencies must be unchanged since that tag. A version with no -tag yet is about to be released from this tree, which is fine. - -Dependencies are compared parsed, not as text, so the release PR's own -version edit does not count. Floors on other workspace packages -(`deepctl-*>=X`) are left out of the comparison: the release sync job raises -them mechanically in packages it is not bumping, and +manifest whose current version already has a release tag, everything the +package ships must be unchanged since that tag. A version with no tag yet is +about to be released from this tree, which is fine. + +What a package ships: its `src/`, and what its pyproject.toml puts in the +wheel (the [project] table, [build-system], [tool.setuptools], and the +[tool.hatch.build] and [tool.hatch.metadata] tables that pick its files). +The root package also ships the npm package, @deepgram/cli, which +release.yml publishes at the root's version: every entry in the `files` of +npm/package.json as committed at HEAD, and package.json itself. A `files` +glob with no slash, like `requirements-*.txt`, matches at any depth under +npm/, as npm matches it. npm/README.md and npm/LICENSE ship too but are left out, as the +packages' READMEs are: a docs change should not force a release. + +pyproject.toml is compared parsed, not as text, so the release PR's own +version edit does not count. Version specifiers on other workspace +packages (`deepctl-*`) are left out of the comparison: the release sync job +raises their floors mechanically in packages it is not bumping, and check_dependency_floors.py already governs them. Adding, removing or -re-pinning any other dependency without a bump is caught. +re-pinning any other dependency, or changing an entry point, without a bump +is caught. npm/package.json is compared as text: release-please writes its +version only on the release PR, when the new version has no tag yet. Tag names follow .github/release-please-config.json: `v` for a package with `include-component-in-tag: false` (the root), otherwise @@ -56,6 +68,16 @@ REPO = Path(__file__).resolve().parent.parent CONFIG = REPO / ".github" / "release-please-config.json" MANIFEST = REPO / ".github" / ".release-please-manifest.json" +NPM_PACKAGE = "npm/package.json" +NPM_DIR = Path(NPM_PACKAGE).parent.as_posix() +GLOB_CHARS = "*?[" +# The [tool.*] tables a build backend reads to decide what goes in the wheel: +# a name maps to None to compare the whole table, or to the subtables that +# count. [tool.hatch.envs] and the like are dev tooling and never ship. +BUILD_TOOLS: dict[str, tuple[str, ...] | None] = { + "setuptools": None, + "hatch": ("build", "metadata"), +} @dataclass(frozen=True) @@ -63,6 +85,7 @@ class Package: path: str # repo-relative, "." for the root version: str tag: str + npm: tuple[str, ...] = () # npm package paths it ships too (the root) @property def src(self) -> str: @@ -72,6 +95,11 @@ def src(self) -> str: def pyproject(self) -> str: return "pyproject.toml" if self.path == "." else f"{self.path}/pyproject.toml" + @property + def shipped(self) -> tuple[str, ...]: + """Repo-relative paths whose content ships; a trailing / is a directory.""" + return (f"{self.src}/", *self.npm) + def dependency_set(pyproject_text: str) -> list[str]: """A package's dependencies, normalized for comparison across releases. @@ -93,6 +121,74 @@ def dependency_set(pyproject_text: str) -> list[str]: return sorted(out) +def shipped_metadata(pyproject_text: str) -> dict[str, Any]: + """What a pyproject.toml puts in the wheel, normalized for comparison. + + The [project] table without `version`, its dependencies as + dependency_set gives them, [build-system], and the build backend's own + tables. [tool.ruff], [tool.mypy] and the like never reach the wheel. + """ + data = tomllib.loads(pyproject_text) + project = { + key: value + for key, value in data.get("project", {}).items() + if key not in ("version", "dependencies", "optional-dependencies") + } + tool = data.get("tool", {}) + build_tools: dict[str, Any] = {} + for name, subtables in BUILD_TOOLS.items(): + if name not in tool: + continue + table = tool[name] + if subtables is not None: + table = {key: table[key] for key in subtables if key in table} + build_tools[name] = table + return { + "project": project, + "dependencies": dependency_set(pyproject_text), + "build-system": data.get("build-system", {}), + "tool": build_tools, + } + + +def npm_paths(repo: Path = REPO) -> tuple[str, ...]: + """The repo paths the npm package ships, from its package.json `files`. + + Read from `files` rather than listed here, so an entry added there is + watched too, and read at HEAD, the commit compared against the tags. + npm always adds package.json itself. Entries keep the spelling they have + in `files`; git_pathspec turns a glob into what git should match. + """ + listed = _git(repo, "ls-tree", "--name-only", "HEAD", "--", NPM_PACKAGE) + if listed.returncode != 0: + raise RuntimeError(f"git ls-tree HEAD failed: {listed.stderr.strip()}") + if not listed.stdout.strip(): + return () + shown = _git(repo, "show", f"HEAD:{NPM_PACKAGE}") + if shown.returncode != 0: + raise RuntimeError( + f"git show HEAD:{NPM_PACKAGE} failed: {shown.stderr.strip()}" + ) + files = json.loads(shown.stdout).get("files", []) + return (*(f"{NPM_DIR}/{entry}" for entry in files), NPM_PACKAGE) + + +def git_pathspec(path: str) -> str: + """The git pathspec for a shipped path. + + A plain path is a literal prefix. A glob gets `:(glob)` magic, and an + npm `files` glob with no slash (`requirements-*.txt`) matches at any + depth, as it does for npm: `**/` in glob magic also matches no + directory, so the top level is still covered. + """ + if not any(c in path for c in GLOB_CHARS): + return path + entry = path.removeprefix(f"{NPM_DIR}/") + if entry != path and "/" not in entry: + return f":(glob){NPM_DIR}/**/{entry}" + return f":(glob){path}" + + def tag_name(config: dict[str, Any], path: str, version: str) -> str: """The tag release-please gives `path` at `version`.""" entry = config.get("packages", {}).get(path, {}) @@ -115,7 +211,8 @@ def packages(repo: Path = REPO) -> list[Package]: version = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"][ "version" ] - out.append(Package(path, version, tag_name(config, path, version))) + npm = npm_paths(repo) if path == "." else () + out.append(Package(path, version, tag_name(config, path, version), npm)) return out @@ -123,23 +220,26 @@ def unbumped( pkgs: list[Package], tag_exists: Callable[[str], bool], changed_since: Callable[[str, str], bool], - deps_changed: Callable[[str, Package], bool] = lambda _tag, _pkg: False, + metadata_changed: Callable[[str, Package], bool] = lambda _tag, _pkg: False, ) -> list[str]: - """Problems for packages whose src/ or dependencies moved past their tag.""" + """Problems for packages whose shipped files or metadata moved past their tag.""" problems = [] for pkg in pkgs: if not tag_exists(pkg.tag): continue # This version has not been released yet. - if changed_since(pkg.tag, pkg.src): - problems.append( - f"{pkg.path}: {pkg.src}/ changed since {pkg.tag}, but the version is" - f" still {pkg.version}, so PyPI would keep serving the old code" - ) - if deps_changed(pkg.tag, pkg): + for path in pkg.shipped: + if changed_since(pkg.tag, path): + registry = "npm" if path.startswith("npm/") else "PyPI" + problems.append( + f"{pkg.path}: {path} changed since {pkg.tag}, but the version is" + f" still {pkg.version}, so {registry} would keep serving the old code" + ) + if metadata_changed(pkg.tag, pkg): problems.append( - f"{pkg.path}: dependencies in {pkg.pyproject} changed since {pkg.tag}," - f" but the version is still {pkg.version}, so PyPI would keep" - " serving the old dependency list" + f"{pkg.path}: packaging metadata in {pkg.pyproject} (dependencies," + f" entry points or build settings) changed since {pkg.tag}, but the" + f" version is still {pkg.version}, so PyPI would keep serving the old" + " metadata" ) return problems @@ -172,7 +272,7 @@ def exists(tag: str) -> bool: def git_changed_since(repo: Path) -> Callable[[str, str], bool]: def changed(tag: str, path: str) -> bool: - result = _git(repo, "diff", "--quiet", tag, "HEAD", "--", path) + result = _git(repo, "diff", "--quiet", tag, "HEAD", "--", git_pathspec(path)) if result.returncode not in (0, 1): raise RuntimeError(f"git diff {tag} failed: {result.stderr.strip()}") return result.returncode == 1 @@ -180,7 +280,7 @@ def changed(tag: str, path: str) -> bool: return changed -def git_deps_changed(repo: Path) -> Callable[[str, Package], bool]: +def git_metadata_changed(repo: Path) -> Callable[[str, Package], bool]: def changed(tag: str, pkg: Package) -> bool: old = _git(repo, "show", f"{tag}:{pkg.pyproject}") new = _git(repo, "show", f"HEAD:{pkg.pyproject}") @@ -189,7 +289,7 @@ def changed(tag: str, pkg: Package) -> bool: raise RuntimeError( f"git show {ref}:{pkg.pyproject} failed: {result.stderr.strip()}" ) - return dependency_set(old.stdout) != dependency_set(new.stdout) + return shipped_metadata(old.stdout) != shipped_metadata(new.stdout) return changed @@ -199,7 +299,7 @@ def main(repo: Path = REPO) -> int: packages(repo), git_tag_exists(repo), git_changed_since(repo), - git_deps_changed(repo), + git_metadata_changed(repo), ) if problems: print("release bump check FAILED:", file=sys.stderr) @@ -212,7 +312,7 @@ def main(repo: Path = REPO) -> int: ) return 1 print( - "release bumps OK: no package ships changed code or dependencies under an old version" + "release bumps OK: no package ships changed code or metadata under an old version" ) return 0 diff --git a/tests/unit/test_release_bumps.py b/tests/unit/test_release_bumps.py index bffe8985..e766f029 100644 --- a/tests/unit/test_release_bumps.py +++ b/tests/unit/test_release_bumps.py @@ -80,12 +80,94 @@ def changed(tag: str, path: str) -> bool: assert gate.unbumped([CORE], lambda tag: False, changed) == [] -def test_only_src_is_compared() -> None: +def test_only_shipped_paths_are_compared() -> None: seen = [] + root = gate.Package(".", "0.4.0", "v0.4.0", ("npm/lib/", "npm/package.json")) gate.unbumped( - [CORE, ROOT], lambda tag: True, lambda tag, path: seen.append(path) or False + [CORE, root], lambda tag: True, lambda tag, path: seen.append(path) or False ) - assert seen == ["packages/deepctl-core/src", "src"] + assert seen == [ + "packages/deepctl-core/src/", + "src/", + "npm/lib/", + "npm/package.json", + ] + + +def test_the_root_ships_the_npm_package_files() -> None: + root = next(pkg for pkg in gate.packages(REPO) if pkg.path == ".") + assert root.npm == ( + "npm/bin/", + "npm/lib/", + "npm/requirements-*.txt", + "npm/package.json", + ) + core = next( + pkg for pkg in gate.packages(REPO) if pkg.path == "packages/deepctl-core" + ) + assert core.npm == () + + +def test_a_changed_npm_file_names_npm() -> None: + root = gate.Package(".", "0.4.0", "v0.4.0", ("npm/lib/",)) + problems = gate.unbumped( + [root], lambda tag: True, lambda tag, path: path == "npm/lib/" + ) + assert problems == [ + ".: npm/lib/ changed since v0.4.0, but the version is still 0.4.0," + " so npm would keep serving the old code" + ] + + +def test_npm_globs_without_a_slash_match_at_any_depth() -> None: + # npm reads `requirements-*.txt` like .gitignore: any depth under npm/. + assert ( + gate.git_pathspec("npm/requirements-*.txt") + == ":(glob)npm/**/requirements-*.txt" + ) + # An entry with a slash is anchored, and plain paths stay literal. + assert gate.git_pathspec("npm/lib/*.js") == ":(glob)npm/lib/*.js" + assert gate.git_pathspec("npm/bin/") == "npm/bin/" + + +@pytest.mark.skipif(shutil.which("git") is None, reason="needs git") +def test_an_npm_glob_matches_top_level_and_nested_files(tmp_path: Path) -> None: + def git(*args: str) -> None: + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=t", + "-c", + "user.email=t@t", + *args, + ], + check=True, + capture_output=True, + ) + + (tmp_path / "npm").mkdir() + (tmp_path / "npm" / "package.json").write_text("{}", encoding="utf-8") + git("init", "-q") + git("add", "-A") + git("commit", "-qm", "release") + git("tag", "v0.4.0") + changed = gate.git_changed_since(tmp_path) + spec = "npm/requirements-*.txt" + for rel, expected in [ + ("npm/notes.txt", False), + ("npm/requirements-0.4.0.txt", True), + ("npm/vendor/requirements-0.4.0.txt", True), + ]: + git("reset", "-q", "--hard", "v0.4.0") + target = tmp_path / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("x\n", encoding="utf-8") + git("add", "-A") + git("commit", "-qm", f"add {rel}") + assert changed("v0.4.0", spec) is expected, rel def pyproject_text(version: str, deps: list[str], optional: dict | None = None) -> str: @@ -131,17 +213,87 @@ def test_optional_dependencies_count() -> None: ) -def test_a_dependency_change_under_a_released_version_fails() -> None: +def test_a_metadata_change_under_a_released_version_fails() -> None: problems = gate.unbumped( [CORE], lambda tag: True, lambda tag, path: False, lambda tag, pkg: True ) assert problems == [ - "packages/deepctl-core: dependencies in packages/deepctl-core/pyproject.toml changed" - " since deepctl-core-v0.3.0, but the version is still 0.3.0, so PyPI would keep" - " serving the old dependency list" + "packages/deepctl-core: packaging metadata in packages/deepctl-core/pyproject.toml" + " (dependencies, entry points or build settings) changed since" + " deepctl-core-v0.3.0, but the version is still 0.3.0, so PyPI would keep" + " serving the old metadata" ] +BASE = """\ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "deepctl" +version = "{version}" +requires-python = ">=3.10" +dependencies = ["click>=8", "deepctl-core>={floor}"] + +[project.scripts] +dg = "deepctl.main:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = {line_length} +""" + + +def base(version: str = "0.4.0", floor: str = "0.3.0", line_length: int = 88) -> str: + return BASE.format(version=version, floor=floor, line_length=line_length) + + +def test_version_floors_and_lint_settings_are_not_shipped_metadata() -> None: + assert gate.shipped_metadata(base()) == gate.shipped_metadata( + base(version="0.4.1", floor="0.4.0", line_length=100) + ) + + +@pytest.mark.parametrize( + ("old", "new"), + [ + ('dg = "deepctl.main:main"', 'dg = "deepctl.cli:main"'), # entry point + ('requires-python = ">=3.10"', 'requires-python = ">=3.11"'), + ('where = ["src"]', 'where = ["lib"]'), # which files go in the wheel + ('"setuptools>=61"', '"setuptools>=70"'), # build backend + ('"click>=8"', '"click>=9"'), # still a dependency change + ], +) +def test_shipped_metadata_changes_are_caught(old: str, new: str) -> None: + assert old in BASE + assert gate.shipped_metadata(base()) != gate.shipped_metadata( + base().replace(old, new) + ) + + +HATCH = """\ +[project] +name = "deepctl" +version = "0.4.0" + +[tool.hatch.build.targets.wheel] +packages = ["src/deepctl"] + +[tool.hatch.envs.default] +dependencies = ["pytest"] +""" + + +def test_only_hatch_build_tables_are_shipped_metadata() -> None: + dev_env = HATCH.replace('["pytest"]', '["pytest", "coverage"]') + assert gate.shipped_metadata(HATCH) == gate.shipped_metadata(dev_env) + wheel = HATCH.replace('["src/deepctl"]', '["src/deepctl", "src/extra"]') + assert gate.shipped_metadata(HATCH) != gate.shipped_metadata(wheel) + + @pytest.mark.skipif(shutil.which("git") is None, reason="needs git") def test_against_a_real_git_repository(tmp_path: Path) -> None: """The CLI end to end: tags, a bump, and a forgotten bump.""" @@ -208,9 +360,8 @@ def run() -> subprocess.CompletedProcess[str]: git("commit", "-qm", "chore: add httpx") result = run() assert result.returncode == 1 - assert ( - "dependencies in packages/deepctl-core/pyproject.toml changed" in result.stderr - ) + assert "packaging metadata in packages/deepctl-core/pyproject.toml" in result.stderr + assert "changed since deepctl-core-v0.3.0" in result.stderr # Code changed under the released version too: the 0.3.0 trap. write("packages/deepctl-core/src/deepctl_core/launcher.py", "X = 1\n") @@ -230,6 +381,127 @@ def run() -> subprocess.CompletedProcess[str]: assert run().returncode == 0, run().stderr +@pytest.mark.skipif(shutil.which("git") is None, reason="needs git") +def test_the_npm_package_and_root_metadata_against_a_real_git_repository( + tmp_path: Path, +) -> None: + """The root's npm files and pyproject metadata need a root bump too.""" + + def git(*args: str) -> None: + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=t", + "-c", + "user.email=t@t", + *args, + ], + check=True, + capture_output=True, + ) + + def write(rel: str, text: str) -> None: + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def commit(message: str) -> None: + git("add", "-A") + git("commit", "-qm", message) + + def run() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), str(tmp_path)], capture_output=True, text=True + ) + + def package_json(version: str, engines: str = ">=20") -> str: + return json.dumps( + { + "name": "@deepgram/cli", + "version": version, + "files": ["bin/", "lib/", "requirements-*.txt"], + "engines": {"node": engines}, + } + ) + + root_only = {"packages": {".": CONFIG["packages"]["."]}} + write(".github/release-please-config.json", json.dumps(root_only)) + write(".github/.release-please-manifest.json", json.dumps({".": "0.4.0"})) + write("pyproject.toml", base()) + write("src/deepctl/__init__.py", "") + write("npm/package.json", package_json("0.4.0")) + write("npm/bin/dg.js", "require('../lib/launcher');\n") + write("npm/lib/launcher.js", "module.exports = {};\n") + git("init", "-q") + commit("release") + git("tag", "v0.4.0") + assert run().returncode == 0, run().stderr + + # npm docs and the npm package's tests do not ship as code: no bump needed. + write("npm/README.md", "docs\n") + write("npm/test/launcher.test.js", "// test\n") + commit("docs(npm): readme") + assert run().returncode == 0, run().stderr + + # A chore to the launcher under the released version: the B3 trap. + write("npm/lib/launcher.js", "module.exports = { x: 1 };\n") + commit("chore(npm): tidy the launcher") + result = run() + assert result.returncode == 1 + assert "npm/lib/ changed since v0.4.0" in result.stderr + assert "so npm would keep serving the old code" in result.stderr + git("revert", "--no-edit", "HEAD") + assert run().returncode == 0, run().stderr + + # The launcher's entry script, under bin/. + write("npm/bin/dg.js", "require('../lib/launcher').main();\n") + commit("chore(npm): call main") + result = run() + assert result.returncode == 1 + assert "npm/bin/ changed since v0.4.0" in result.stderr + git("revert", "--no-edit", "HEAD") + assert run().returncode == 0, run().stderr + + # A committed requirements file matches the `files` glob, so it ships. + write("npm/requirements-0.4.0.txt", "deepctl==0.4.0\n") + commit("chore(npm): pin requirements") + result = run() + assert result.returncode == 1 + assert "npm/requirements-*.txt changed since v0.4.0" in result.stderr + git("revert", "--no-edit", "HEAD") + assert run().returncode == 0, run().stderr + + # A package.json change other than the version. + write("npm/package.json", package_json("0.4.0", engines=">=22")) + commit("chore(npm): raise engines") + result = run() + assert result.returncode == 1 + assert "npm/package.json changed since v0.4.0" in result.stderr + git("revert", "--no-edit", "HEAD") + assert run().returncode == 0, run().stderr + + # A root entry point changed under the released version. + write("pyproject.toml", base().replace("deepctl.main:main", "deepctl.cli:main")) + commit("refactor: move main") + result = run() + assert result.returncode == 1 + assert "packaging metadata in pyproject.toml" in result.stderr + + # The release PR: root version and npm/package.json bumped together. The + # new version has no tag yet, so everything above ships with it. + write( + "pyproject.toml", + base(version="0.4.1").replace("deepctl.main:main", "deepctl.cli:main"), + ) + write("npm/package.json", package_json("0.4.1")) + write("npm/lib/launcher.js", "module.exports = { x: 2 };\n") + commit("chore: release 0.4.1") + assert run().returncode == 0, run().stderr + + def test_a_repository_without_tags_fails_instead_of_passing(tmp_path: Path) -> None: """A shallow checkout has no tags; that must not read as "first release".""" if shutil.which("git") is None: From e6232e51b93769ee00d127bdb65d548fa6f17f7e Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Tue, 10 Nov 2026 09:36:48 -0500 Subject: [PATCH 15/15] ci(release): deploy web only after npm publish and fix npm prerequisite copy --- .github/workflows/release.yml | 6 +++--- CONTRIBUTING.md | 15 ++++++++++----- README.md | 4 ++-- web/src/data/llms-full.txt | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fcdec38d..3f672064 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -312,10 +312,10 @@ jobs: deploy-web: name: Deploy web to production - needs: [release-please, publish, verify-published] + needs: [release-please, publish, verify-published, publish-npm] # Only fire on root-package releases (v0.2.4, v1.0.0, …) and only after - # PyPI publish so cli.deepgram.com never advertises a version that isn't - # installable yet. Sub-package tags look like deepctl-cmd-listen-v0.0.3 — + # PyPI publish and the npm publish + npx smoke test, so cli.deepgram.com + # never advertises a version that isn't installable yet. Sub-package tags look like deepctl-cmd-listen-v0.0.3 — # skip those. if: | needs.release-please.outputs.release_created == 'true' && diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c2e2239..b9ed1515 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,11 +103,16 @@ dedicated PAT or GitHub App token. **Nothing advertises a release until it is installable.** `verify-published` polls PyPI until `pip install deepctl==X` resolves its full closure, then -installs it for real and runs the CLI. `mark-latest`, `deploy-web`, and the -Homebrew bump all wait on it. If it times out, PyPI has not propagated or a -sub-package failed to publish — re-run all jobs on that workflow run once -PyPI has caught up; re-running only the failed job leaves the three -downstream jobs skipped. Re-running all jobs is safe: the publish step uses +installs it for real and runs the CLI. `mark-latest`, `deploy-web`, the +Homebrew bump, and `publish-npm` all wait on it. `deploy-web` also waits on +`publish-npm`, which publishes `@deepgram/cli` and smoke-tests it with `npx`, +so the site never shows npm install instructions before the package works. +If `verify-published` times out, PyPI has not propagated or a sub-package +failed to publish — re-run all jobs on that workflow run once PyPI has caught +up; re-running only the failed job leaves the downstream jobs skipped. If +only `publish-npm` fails, fix the cause and re-run failed jobs: `deploy-web` +stays skipped until it passes, and the npm publish step skips an +already-published version. Re-running all jobs is safe: the publish step uses `skip-existing`. ### Testing diff --git a/README.md b/README.md index 74fb5153..a2db7cc6 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ npm install -g @deepgram/cli # npm (installs dg and deepgram) ``` The npm package is a launcher for the same Python CLI. It needs -[uv](https://docs.astral.sh/uv/) or Python 3.10+ with `venv`, and sets the CLI -up on first run. See [npm/README.md](npm/README.md). +Node.js 20+ and either [uv](https://docs.astral.sh/uv/) or Python 3.10-3.14 +with `venv`, and sets the CLI up on first run. See [npm/README.md](npm/README.md). ### Try Without Installing diff --git a/web/src/data/llms-full.txt b/web/src/data/llms-full.txt index b0019ad2..b443daf0 100644 --- a/web/src/data/llms-full.txt +++ b/web/src/data/llms-full.txt @@ -24,7 +24,7 @@ pip install deepctl # pip pipx install deepctl # pipx (isolated) uvx deepctl --help # uvx (run without install) brew install deepgram/tap/deepgram # Homebrew -npm install -g @deepgram/cli # npm (needs uv or Python 3.10+; installs dg, deepgram) +npm install -g @deepgram/cli # npm (needs Node.js 20+ and uv or Python 3.10-3.14 with venv; installs dg, deepgram) npx @deepgram/cli --help # npx (run without install) ```