diff --git a/conftest.py b/conftest.py index c94232e4..86d564bc 100644 --- a/conftest.py +++ b/conftest.py @@ -55,6 +55,19 @@ def _clean_deepgram_env(monkeypatch): monkeypatch.delenv(key, raising=False) +@pytest.fixture(autouse=True) +def _speak_tests_run_as_a_person(request, monkeypatch): + """Speak tests run as a person by default: the agent rules key off hard + agent markers, which a developer's own shell may set (CLAUDECODE=1). + Tests that need agent mode patch ``is_ai_agent`` themselves.""" + if "deepctl-cmd-speak" not in request.path.parts: + return + from deepctl_core.output import AI_AGENT_ENV_MARKERS + + for name in (*AI_AGENT_ENV_MARKERS, "OR_APP_NAME", "OR_SITE_URL"): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture(autouse=True) def _disable_telemetry(monkeypatch): """Keep tests that call `main()` from initializing real telemetry. @@ -64,3 +77,14 @@ def _disable_telemetry(monkeypatch): """ monkeypatch.setenv("DEEPCTL_TELEMETRY_DISABLED", "1") monkeypatch.setenv("DEEPCTL_TELEMETRY_DSN", "http://k@127.0.0.1:9/1") + + +@pytest.fixture(autouse=True) +def _restore_sigint(): + """`main()` installs its own SIGINT handler; put pytest's back.""" + import signal + + before = signal.getsignal(signal.SIGINT) + yield + if before is not None: + signal.signal(signal.SIGINT, before) diff --git a/packages/deepctl-cmd-debug-audio/README.md b/packages/deepctl-cmd-debug-audio/README.md index 832eb13c..ac7a1d16 100644 --- a/packages/deepctl-cmd-debug-audio/README.md +++ b/packages/deepctl-cmd-debug-audio/README.md @@ -38,7 +38,6 @@ pipx run deepctl --help - `click>=8.2.0,<9` - `rich>=13.0.0,<16` - `pydantic>=2.0.0,<3` -- `ffmpeg-python>=0.2.0` - `httpx>=0.24.0,<1` ## License diff --git a/packages/deepctl-cmd-debug-audio/pyproject.toml b/packages/deepctl-cmd-debug-audio/pyproject.toml index 57061a62..0aba4d52 100644 --- a/packages/deepctl-cmd-debug-audio/pyproject.toml +++ b/packages/deepctl-cmd-debug-audio/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "click>=8.2.0,<9", "rich>=13.0.0,<16", "pydantic>=2.0.0,<3", - "ffmpeg-python>=0.2.0", "httpx>=0.24.0,<1", ] diff --git a/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py b/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py index aec07b51..710db3d8 100644 --- a/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py +++ b/packages/deepctl-cmd-debug-audio/src/deepctl_cmd_debug_audio/command.py @@ -8,7 +8,6 @@ from typing import Any from urllib.parse import urlparse -import ffmpeg # type: ignore[import-untyped] from deepctl_core import ( AuthManager, BaseCommand, @@ -45,6 +44,32 @@ def _ffprobe_input(file_path: str) -> str: console = get_console() status_console = get_status_console() +# Seconds a single ffprobe run may take before it is abandoned. +FFPROBE_TIMEOUT_S = 30 + + +def _probe_input(path: str) -> list[str]: + """ffprobe's input arguments, always last on the command line. + + The input is always a local file (URLs are downloaded first), so ffprobe + may open it only through the file protocol: tcp:, concat:, pipe: and the + rest are refused. It is named as ``file:`` + (``_ffprobe_input``), so ffprobe reads it as a file and never as an + option, whatever it is called. + """ + return ["-protocol_whitelist", "file", _ffprobe_input(path)] + + +def _run_ffprobe(cmd: list[str]) -> "subprocess.CompletedProcess[bytes]": + """Run ffprobe from an argv list: no shell, no stdin, bounded in time.""" + return subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + capture_output=True, + timeout=FFPROBE_TIMEOUT_S, + check=False, + ) + def _human_output() -> bool: """True when stdout is for a person, not a parser.""" @@ -239,17 +264,37 @@ def run_ffprobe( # value stays one argument, and run ffprobe directly cmd = [ffprobe_bin, "-v", "quiet", "-print_format", "json"] cmd.extend(shlex.split(custom_args)) - cmd.append(_ffprobe_input(file_path)) + cmd.extend(_probe_input(file_path)) - result = subprocess.run(cmd, capture_output=True, text=True) + result = _run_ffprobe(cmd) if result.returncode != 0: - raise Exception(f"ffprobe failed: {result.stderr}") - - return dict(json.loads(result.stdout)) - else: - # Use python-ffmpeg for standard probe - probe = ffmpeg.probe(_ffprobe_input(file_path), cmd=ffprobe_bin) - return dict(probe) + stderr = result.stderr.decode("utf-8", errors="replace") + raise Exception(f"ffprobe failed: {stderr}") + + return dict(json.loads(result.stdout.decode("utf-8"))) + # The standard probe: the argv ffmpeg-python's probe() built, + # plus the file-only protocol whitelist. + cmd = [ + ffprobe_bin, + "-show_format", + "-show_streams", + "-of", + "json", + *_probe_input(file_path), + ] + probe = _run_ffprobe(cmd) + if probe.returncode != 0: + # stderr is captured, so say what ffprobe said: its last line. + said = [ + line.strip() + for line in (probe.stderr or b"") + .decode("utf-8", "replace") + .splitlines() + if line.strip() + ] + why = said[-1][:300] if said else f"exit {probe.returncode}" + raise Exception(f"ffprobe couldn't read the file: {why}") + return dict(json.loads(probe.stdout.decode("utf-8"))) except Exception as e: raise Exception(f"Failed to probe file: {e!s}") diff --git a/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py b/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py index e8680cd9..fd9ac68b 100644 --- a/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py +++ b/packages/deepctl-cmd-debug-audio/tests/unit/test_audio_command.py @@ -2,9 +2,15 @@ import json import os +import shutil +import socket +import subprocess +import threading +import time from unittest.mock import MagicMock, Mock, patch import pytest +from deepctl_cmd_debug_audio import command as command_module from deepctl_cmd_debug_audio.command import AudioCommand from deepctl_cmd_debug_audio.models import ( AudioDebugResult, @@ -12,7 +18,6 @@ AudioInfo, AudioStream, ) -from deepctl_cmd_debug_audio import command as command_module from deepctl_core import AuthManager, Config, DeepgramClient @@ -26,6 +31,15 @@ def _ffprobe_lookup_via_which(monkeypatch): ) +def _completed(returncode: int, stdout: str, stderr: bytes = b"") -> Mock: + """A subprocess.run result carrying ffprobe's bytes output.""" + result = Mock() + result.returncode = returncode + result.stdout = stdout.encode("utf-8") + result.stderr = stderr + return result + + class TestAudioCommand: """Test cases for AudioCommand.""" @@ -109,30 +123,103 @@ def test_check_ffmpeg_installed(self, mock_which, command): assert command.check_ffmpeg_installed() is False @patch("shutil.which", return_value="/usr/bin/ffprobe") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_run_ffprobe_standard( - self, mock_probe, mock_which, command, sample_probe_data + self, mock_run, mock_which, command, sample_probe_data ): - """Test running ffprobe without custom arguments.""" - mock_probe.return_value = sample_probe_data + """The standard probe runs ffprobe's JSON argv as a list, no shell.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.run_ffprobe("test.mp3") assert result == sample_probe_data - mock_probe.assert_called_once_with( - "file:" + os.path.abspath("test.mp3"), cmd="/usr/bin/ffprobe" + mock_run.assert_called_once() + args, kwargs = mock_run.call_args + assert args[0] == [ + "/usr/bin/ffprobe", + "-show_format", + "-show_streams", + "-of", + "json", + "-protocol_whitelist", + "file", + "file:" + os.path.abspath("test.mp3"), + ] + assert kwargs.get("shell", False) is False + assert kwargs["timeout"] == 30 + assert kwargs["capture_output"] is True + assert kwargs["stdin"] is subprocess.DEVNULL + + @pytest.mark.parametrize( + ("given", "probed"), + [ + ("-v.wav", "file:" + os.path.abspath("-v.wav")), + ("-show_entries.wav", "file:" + os.path.abspath("-show_entries.wav")), + ("/abs/-v.wav", "file:" + os.path.abspath("/abs/-v.wav")), + ("dir/-v.wav", "file:" + os.path.abspath("dir/-v.wav")), + ], + ) + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_never_passes_a_path_as_an_option( + self, mock_run, mock_which, command, sample_probe_data, given, probed + ): + """A file name that starts with "-" reaches ffprobe as a file.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) + + command.run_ffprobe(given) + assert mock_run.call_args[0][0][-3:] == ["-protocol_whitelist", "file", probed] + + command.run_ffprobe(given, "-show_streams") + assert mock_run.call_args[0][0][-3:] == ["-protocol_whitelist", "file", probed] + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_failure_says_what_ffprobe_said( + self, mock_run, mock_which, command + ): + """A non-zero ffprobe exit shows the last line ffprobe wrote.""" + mock_run.return_value = _completed( + 1, "", b"ffprobe version 7\n\nbroken.mp4: moov atom not found\n" ) + with pytest.raises(Exception) as exc: + command.run_ffprobe("broken.mp4") + + assert str(exc.value) == ( + "Failed to probe file: ffprobe couldn't read the file: " + "broken.mp4: moov atom not found" + ) + + mock_run.return_value = _completed(1, "", b"") + with pytest.raises(Exception) as exc: + command.run_ffprobe("broken.mp4") + assert str(exc.value).endswith("ffprobe couldn't read the file: exit 1") + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_timeout_is_a_probe_failure( + self, mock_run, mock_which, command + ): + """An ffprobe that hangs past the timeout is reported, not waited on.""" + import subprocess + + mock_run.side_effect = subprocess.TimeoutExpired(["ffprobe"], 30) + + with pytest.raises(Exception) as exc: + command.run_ffprobe("slow.mp4") + + assert str(exc.value).startswith("Failed to probe file: ") + assert "timed out after 30 seconds" in str(exc.value) + @patch("shutil.which", return_value="/usr/bin/ffprobe") @patch("subprocess.run") def test_run_ffprobe_custom_args( self, mock_run, mock_which, command, sample_probe_data ): - """Test running ffprobe with custom arguments.""" - mock_result = Mock() - mock_result.returncode = 0 - mock_result.stdout = json.dumps(sample_probe_data) - mock_run.return_value = mock_result + """Custom arguments go before the input, which keeps the file-only + whitelist, the timeout and the closed stdin.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.run_ffprobe("test.mp3", "-show_streams -show_format") @@ -145,23 +232,44 @@ def test_run_ffprobe_custom_args( "json", "-show_streams", "-show_format", + "-protocol_whitelist", + "file", "file:" + os.path.abspath("test.mp3"), ] mock_run.assert_called_once() assert mock_run.call_args[0][0] == expected_cmd + assert mock_run.call_args[1]["timeout"] == 30 + assert mock_run.call_args[1]["stdin"] is subprocess.DEVNULL + + @patch("shutil.which", return_value="/usr/bin/ffprobe") + @patch("subprocess.run") + def test_run_ffprobe_custom_args_cannot_widen_the_whitelist( + self, mock_run, mock_which, command, sample_probe_data + ): + """A user-supplied -protocol_whitelist comes before ours, so the + file-only one is the one that applies to the input.""" + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) + + command.run_ffprobe("test.mp3", "-protocol_whitelist file,tcp") + + assert mock_run.call_args[0][0][-3:] == [ + "-protocol_whitelist", + "file", + "file:" + os.path.abspath("test.mp3"), + ] @patch("shutil.which", return_value="/usr/bin/ffprobe") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_option_like_file_name_is_probed_as_a_file( - self, mock_probe, mock_which, command, sample_probe_data, tmp_path, monkeypatch + self, mock_run, mock_which, command, sample_probe_data, tmp_path, monkeypatch ): """A file named "-f lavfi" is an input, not options.""" - mock_probe.return_value = sample_probe_data + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) monkeypatch.chdir(tmp_path) command.run_ffprobe("-f lavfi") - target = mock_probe.call_args.args[0] + target = mock_run.call_args[0][0][-1] assert target == f"file:{tmp_path}{os.sep}-f lavfi" assert not target.startswith("-") @@ -171,7 +279,7 @@ def test_custom_args_are_split_like_a_shell( self, mock_run, mock_which, command, sample_probe_data ): """--ffprobe-args keeps a quoted value as one argument (shlex).""" - mock_run.return_value = Mock(returncode=0, stdout=json.dumps(sample_probe_data)) + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) command.run_ffprobe( "-f lavfi", "-show_entries 'format=duration,size' -select_streams a" @@ -183,6 +291,8 @@ def test_custom_args_are_split_like_a_shell( "format=duration,size", "-select_streams", "a", + "-protocol_whitelist", + "file", "file:" + os.path.abspath("-f lavfi"), ] @@ -235,10 +345,10 @@ def test_handle_ffmpeg_not_installed( assert result.ffmpeg_installed is False @patch("shutil.which") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_handle_success( self, - mock_probe, + mock_run, mock_which, command, mock_config, @@ -248,7 +358,7 @@ def test_handle_success( ): """Test successful audio file analysis.""" mock_which.return_value = "/usr/bin/ffprobe" - mock_probe.return_value = sample_probe_data + mock_run.return_value = _completed(0, json.dumps(sample_probe_data)) result = command.handle( config=mock_config, @@ -264,10 +374,10 @@ def test_handle_success( assert result.audio_info.format.filename == "test.mp3" @patch("shutil.which") - @patch("ffmpeg.probe") + @patch("subprocess.run") def test_handle_probe_error( self, - mock_probe, + mock_run, mock_which, command, mock_config, @@ -276,7 +386,7 @@ def test_handle_probe_error( ): """Test handling ffprobe errors.""" mock_which.return_value = "/usr/bin/ffprobe" - mock_probe.side_effect = Exception("Failed to probe file") + mock_run.return_value = _completed(1, "", b"Invalid data found") result = command.handle( config=mock_config, @@ -382,9 +492,7 @@ def test_analysis_failure_panel_goes_to_stderr( def test_human_rendering_is_suppressed_for_machines( self, _ffmpeg, _format, command, mocks, probe_data, capsys ): - with patch.object( - AudioCommand, "run_ffprobe", return_value=probe_data - ): + with patch.object(AudioCommand, "run_ffprobe", return_value=probe_data): result = command.handle(*mocks, file="test.mp3") captured = capsys.readouterr() @@ -392,16 +500,12 @@ def test_human_rendering_is_suppressed_for_machines( assert captured.out == "" assert "Analyzing audio file" in captured.err - @patch( - "deepctl_cmd_debug_audio.command.get_output_format", return_value="default" - ) + @patch("deepctl_cmd_debug_audio.command.get_output_format", return_value="default") @patch.object(AudioCommand, "check_ffmpeg_installed", return_value=True) def test_human_rendering_still_prints_for_humans( self, _ffmpeg, _format, command, mocks, probe_data, capsys ): - with patch.object( - AudioCommand, "run_ffprobe", return_value=probe_data - ): + with patch.object(AudioCommand, "run_ffprobe", return_value=probe_data): command.handle(*mocks, file="test.mp3") captured = capsys.readouterr() @@ -423,3 +527,81 @@ def test_ffprobe_never_from_current_directory(tmp_path, monkeypatch): assert command.check_ffmpeg_installed() is False with pytest.raises(Exception, match="ffprobe not found"): command.run_ffprobe("a.wav") + + +# -- real ffprobe: the argv hardening holds against the binary itself ------- + +_FFPROBE = shutil.which("ffprobe") +needs_ffprobe = pytest.mark.skipif(_FFPROBE is None, reason="ffprobe not installed") + + +def _write_wav(path) -> None: + import wave + + with wave.open(str(path), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(16000) + w.writeframes(b"\x00\x00" * 1600) + + +@pytest.fixture +def real_command(monkeypatch): + """An AudioCommand that uses the real ffprobe on PATH.""" + monkeypatch.setattr(command_module, "resolve_tool", lambda *a: _FFPROBE) + return AudioCommand() + + +@needs_ffprobe +class TestRealFfprobe: + def test_dash_named_file_is_probed_as_a_file( + self, real_command, tmp_path, monkeypatch + ): + monkeypatch.chdir(tmp_path) + _write_wav(tmp_path / "-v.wav") + + for custom in (None, "-show_streams"): + data = real_command.run_ffprobe("-v.wav", custom) + assert data["streams"][0]["codec_name"] == "pcm_s16le" + + def test_tcp_input_is_refused_without_connecting(self, real_command): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(0.2) + port = listener.getsockname()[1] + accepted: list[object] = [] + stop = threading.Event() + + def accept() -> None: + while not stop.is_set(): + try: + accepted.append(listener.accept()[0]) + except OSError: + continue + + thread = threading.Thread(target=accept, daemon=True) + thread.start() + try: + for custom in (None, "-show_streams", "-protocol_whitelist file,tcp"): + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe(f"tcp://127.0.0.1:{port}", custom) + time.sleep(0.3) + finally: + stop.set() + thread.join(2) + listener.close() + assert accepted == [] + + def test_concat_input_is_refused(self, real_command, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write_wav(tmp_path / "a.wav") + + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe("concat:a.wav|a.wav") + + def test_pipe_input_returns_promptly(self, real_command): + start = time.monotonic() + with pytest.raises(Exception, match="^Failed to probe file: "): + real_command.run_ffprobe("pipe:0") + assert time.monotonic() - start < 5 diff --git a/packages/deepctl-cmd-debug-browser/README.md b/packages/deepctl-cmd-debug-browser/README.md index 61b1710f..d31d5043 100644 --- a/packages/deepctl-cmd-debug-browser/README.md +++ b/packages/deepctl-cmd-debug-browser/README.md @@ -39,7 +39,7 @@ pipx run deepctl --help - `rich>=13.0.0,<16` - `pydantic>=2.0.0,<3` - `httpx>=0.24.0,<1` -- `aiohttp>=3.9.0` +- `websockets>=13.1,<18` ## License diff --git a/packages/deepctl-cmd-debug-browser/pyproject.toml b/packages/deepctl-cmd-debug-browser/pyproject.toml index 9be8c267..bef237b3 100644 --- a/packages/deepctl-cmd-debug-browser/pyproject.toml +++ b/packages/deepctl-cmd-debug-browser/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "rich>=13.0.0,<16", "pydantic>=2.0.0,<3", "httpx>=0.24.0,<1", - "aiohttp>=3.9.0", + "websockets>=13.1,<18", ] [project.optional-dependencies] diff --git a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py index f765fa43..781382cd 100644 --- a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py +++ b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/command.py @@ -2,17 +2,26 @@ import importlib.resources import json +import logging +import re import socket import time import webbrowser from pathlib import Path -from typing import Any - -from deepctl_core import AuthManager, BaseCommand, Config, DeepgramClient -from rich.console import Console +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs, urlsplit + +from deepctl_core import ( + AuthManager, + BaseCommand, + Config, + DeepgramClient, + get_status_console, +) from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table +from rich.text import Text from .models import ( BrowserCapabilities, @@ -22,24 +31,111 @@ WebSocketMessage, ) -console = Console() +if TYPE_CHECKING: + from websockets.asyncio.server import Server, ServerConnection + from websockets.http11 import Request, Response + +# Everything this command prints is chrome, and the page link carries the +# session token, so all of it goes to stderr: under -o json or yaml, stdout +# holds only the result. +console = get_status_console() + +PORT_RANGE = range(3000, 3100) +MAX_FRAME_BYTES = 2**20 +# What the page can make this process keep: messages by count and by total +# size, and capability names. +MAX_MESSAGES = 1000 +MAX_MESSAGE_BYTES = 8 * 2**20 # counted as stored JSON +MAX_MESSAGE_TEXT = 64 * 2**10 # a longer frame is dropped +CAPABILITY_KEYS = ( + "web_audio_api", + "audio_context", + "audio_worklet", + "websocket_api", + "fetch_api", + "es6_features", + "dom_apis", + "console_api", + "timer_apis", + "secure_context", + "microphone_access", + "websocket_deepgram", +) +_CONTROL = re.compile(r"[\x00-\x1f\x7f]") +# The same fixed body for every 403: it echoes nothing from the request and +# doesn't say which check failed. +FORBIDDEN = "Forbidden. Open the exact link that dg debug browser printed.\n" +LINK_NOTE = "It works only while this command runs." + +# The page and the WebSocket are only for the local browser this command +# opens: loopback Host, the page's own Origin, and the session token. +PAGE_HEADERS = ( + ("Content-Type", "text/html; charset=utf-8"), + ("Cache-Control", "no-store"), + ("X-Content-Type-Options", "nosniff"), + ("X-Frame-Options", "DENY"), + ("Content-Security-Policy", "frame-ancestors 'none'"), + ("Referrer-Policy", "no-referrer"), +) + + +def _quiet_logger() -> logging.Logger: + """A logger for the server that never writes request headers anywhere, + whatever the root logger's level or handlers.""" + logger = logging.getLogger("deepctl.debug.browser.server") + if not logger.handlers: + logger.addHandler(logging.NullHandler()) + logger.propagate = False + logger.setLevel(logging.CRITICAL + 1) + return logger + + +def _untrusted(text: str, style: str = "") -> Text: + """Text from the page for rich to print as it is: plain ASCII, control + characters and anything non-ASCII as backslash escapes, in a Text object + so neither markup nor :emoji: codes in it are interpreted.""" + text = text.encode("ascii", "backslashreplace").decode("ascii") + return Text(_CONTROL.sub(lambda m: f"\\x{ord(m.group()):02x}", text), style=style) -# aiohttp and asyncio (with ssl) load only when the servers start, so -# `dg debug browser --help` doesn't import them (W3.2). +def _count(n: int, one: str, many: str) -> str: + return f"{n} {one if n == 1 else many}" + + +def _print_link(lead: str, url: str, note: str) -> None: + """The page link, whole on one line so it can be copied, then a dim note. + These lines are the only place the session token is shown (stderr).""" + console.print(f"\n{lead} [cyan]{url}[/cyan]", soft_wrap=True) + console.print(f"[dim]{note}[/dim]", soft_wrap=True) + + +def _respond( + connection: "ServerConnection", + status: int, + text: str, + headers: tuple[tuple[str, str], ...] = (), +) -> "Response": + """A complete response that ends the connection. + + connection.respond() also marks the handshake as refused, so websockets + never runs the WebSocket handler for it (13.x would for a hand-built + Response). + """ + response = connection.respond(status, text) + if headers: + del response.headers["Content-Type"] + for name, value in headers: + response.headers[name] = value + return response + + +# asyncio (with ssl) loads only when the server starts, so +# `dg debug browser --help` doesn't import it. def __getattr__(name: str) -> Any: if name == "asyncio": import asyncio return asyncio - if name == "web": - from aiohttp import web - - return web - if name == "WSMsgType": - from aiohttp import WSMsgType - - return WSMsgType raise AttributeError(f"module {__name__!r} has no attribute {name!r}") @@ -72,6 +168,12 @@ def __init__(self) -> None: self.ws_clients: set[Any] = set() self.debug_complete = False self.start_time: float | None = None + self._token = "" + self._server: Server | None = None + self.browser_opened = False + self._stored_bytes = 0 + self.dropped_messages = 0 + self.dropped_capabilities = 0 def get_arguments(self) -> list[dict[str, Any]]: """Get command arguments and options.""" @@ -111,119 +213,198 @@ def get_arguments(self) -> list[dict[str, Any]]: def find_available_port(self, start_port: int = 3000) -> int: """Find an available port starting from the given port.""" for port in range(start_port, start_port + 100): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.bind(("localhost", port)) - sock.close() + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind(("127.0.0.1", port)) + except OSError: + continue return port - except OSError: - continue raise RuntimeError(f"Could not find available port starting from {start_port}") - async def websocket_handler(self, request: Any) -> Any: - """Handle WebSocket connections from the browser.""" - from aiohttp import WSMsgType, web + @staticmethod + def _allowed(values: list[str], form: str, port: int) -> bool: + """True if there is one value, and it is `form` for 127.0.0.1 or + localhost on `port` (case-insensitive; on port 80 without the port, + as browsers send it).""" + forms = [form] + if port == 80: + forms.append(form.replace(":{port}", "")) + return len(values) == 1 and values[0].lower() in { + f.format(host=host, port=port) + for f in forms + for host in ("127.0.0.1", "localhost") + } - ws = web.WebSocketResponse() - await ws.prepare(request) - self.ws_clients.add(ws) + def _token_ok(self, target: str) -> bool: + import hmac + + tokens = parse_qs(urlsplit(target).query).get("t", []) + return len(tokens) == 1 and hmac.compare_digest( + tokens[0].encode("utf-8", "surrogateescape"), + self._token.encode(), + ) + def _process_request( + self, connection: "ServerConnection", request: "Request" + ) -> "Response | None": + """Serve the page, refuse, or let a /ws upgrade through. + + Runs before any WebSocket handshake. Rejections are fixed text and + never echo the Host, Origin or token back. + """ + # websockets < 17 closes the connection on other methods itself, + # without a response. + if getattr(request, "method", "GET") != "GET": + return _respond(connection, 405, "Method not allowed.\n") + port = connection.local_address[1] + if not self._allowed(request.headers.get_all("Host"), "{host}:{port}", port): + return _respond(connection, 403, FORBIDDEN) + path = urlsplit(request.path).path + if path not in ("/", "/ws"): + return _respond(connection, 404, "Not found.\n") + if not self._token_ok(request.path): + return _respond(connection, 403, FORBIDDEN) + if path == "/": + return _respond(connection, 200, self._page(), PAGE_HEADERS) + origins = request.headers.get_all("Origin") + if not self._allowed(origins, "http://{host}:{port}", port): + return _respond(connection, 403, FORBIDDEN) + return None + + @staticmethod + def _page() -> str: + """The debug page, read from the package.""" try: - async for msg in ws: - if msg.type == WSMsgType.TEXT: - data = json.loads(msg.data) - ws_msg = WebSocketMessage( - type=MessageType(data.get("type", "info")), - data=data.get("data", {}), - message=data.get("message"), - ) - self.messages.append(ws_msg) - - # Handle different message types - if ws_msg.type == MessageType.CAPABILITY_CHECK: - cap_data = ws_msg.data.get("result", {}) - cap_key = ws_msg.data.get("capability") - if cap_key and cap_data: - self.capabilities_data[cap_key] = BrowserCapability( - **cap_data - ) - - elif ws_msg.type == MessageType.COMPLETE: - # Store complete capabilities - if "capabilities" in ws_msg.data: - caps = ws_msg.data["capabilities"] - # Convert all capability data - cap_dict = {} - for key in [ - "web_audio_api", - "audio_context", - "audio_worklet", - "websocket_api", - "fetch_api", - "es6_features", - "dom_apis", - "console_api", - "timer_apis", - "secure_context", - "microphone_access", - "websocket_deepgram", - ]: - if key in caps: - cap_dict[key] = BrowserCapability(**caps[key]) - elif key in self.capabilities_data: - cap_dict[key] = self.capabilities_data[key] - - # Store complete capabilities - self.capabilities_data = { - **cap_dict, - "user_agent": caps.get("user_agent", "Unknown"), - "overall_compatible": caps.get( - "overall_compatible", False - ), - } - self.debug_complete = True - elif msg.type == WSMsgType.ERROR: - console.print(f"[red]WebSocket error: {ws.exception()}[/red]") + files = importlib.resources.files("deepctl_cmd_debug_browser") + html_content = (files / "static" / "debug.html").read_text() + except Exception: + # Fallback to Path-based approach + html_path = Path(__file__).parent / "static" / "debug.html" + html_content = html_path.read_text() + return html_content + + def _handle_text(self, text: str) -> None: + """Record one message from the page. + + Raises ValueError, TypeError or AttributeError for a frame that isn't + a valid message; nothing is recorded then. + """ + if len(text) > MAX_MESSAGE_TEXT: + self.dropped_messages += 1 + return + data = json.loads(text) + ws_msg = WebSocketMessage( + type=MessageType(data.get("type", "info")), + data=data.get("data", {}), + message=data.get("message"), + ) + capabilities = self.capabilities_data + + # Handle different message types + if ws_msg.type == MessageType.CAPABILITY_CHECK: + cap_data = ws_msg.data.get("result", {}) + cap_key = ws_msg.data.get("capability") + if cap_key and cap_key not in CAPABILITY_KEYS: + self.dropped_capabilities += 1 + elif cap_key and cap_data: + capabilities = { + **capabilities, + cap_key: BrowserCapability(**cap_data), + } + + elif ws_msg.type == MessageType.COMPLETE: + # Store complete capabilities + if "capabilities" in ws_msg.data: + caps = ws_msg.data["capabilities"] + # Convert all capability data + cap_dict = {} + for key in CAPABILITY_KEYS: + if key in caps: + cap_dict[key] = BrowserCapability(**caps[key]) + elif key in self.capabilities_data: + cap_dict[key] = self.capabilities_data[key] + + # Store complete capabilities + capabilities = { + **cap_dict, + "user_agent": caps.get("user_agent", "Unknown"), + "overall_compatible": caps.get("overall_compatible", False), + } + + size = len(ws_msg.model_dump_json()) + if ( + len(self.messages) < MAX_MESSAGES + and self._stored_bytes + size <= MAX_MESSAGE_BYTES + ): + self.messages.append(ws_msg) + self._stored_bytes += size + else: + self.dropped_messages += 1 + self.capabilities_data = capabilities + if ws_msg.type == MessageType.COMPLETE: + self.debug_complete = True + + async def websocket_handler(self, ws: "ServerConnection") -> None: + """Handle WebSocket connections from the browser. + + Text frames carry JSON messages; a frame that isn't a valid message + is dropped and the connection stays open. Binary frames are ignored. + """ + from websockets.exceptions import ConnectionClosed + + self.ws_clients.add(ws) + try: + async for frame in ws: + if isinstance(frame, str): + try: + self._handle_text(frame) + except (ValueError, TypeError, AttributeError): + continue + except ConnectionClosed: + pass finally: self.ws_clients.discard(ws) - await ws.close() - return ws + async def _start_server(self, port: int | None) -> int: + """Listen on 127.0.0.1 and return the port. - async def http_handler(self, request: Any) -> Any: - """Serve the debug HTML page.""" - from aiohttp import web + A given port is tried once and an OSError propagates (0 picks any + free port). Otherwise the first port in 3000-3099 that binds is used. + """ + from websockets.asyncio.server import serve - if request.path == "/": - # Read the HTML template from the static directory + for candidate in PORT_RANGE if port is None else [port]: try: - files = importlib.resources.files("deepctl_cmd_debug_browser") - html_content = (files / "static" / "debug.html").read_text() - except Exception: - # Fallback to Path-based approach - html_path = Path(__file__).parent / "static" / "debug.html" - html_content = html_path.read_text() + self._server = await serve( + self.websocket_handler, + "127.0.0.1", + candidate, + process_request=self._process_request, + max_size=MAX_FRAME_BYTES, + ping_interval=None, + server_header=None, + logger=_quiet_logger(), + ) + except OSError: + if port is not None: + raise + continue + bound: int = next(iter(self._server.sockets)).getsockname()[1] + return bound + raise RuntimeError( + f"Could not find available port starting from {PORT_RANGE.start}" + ) - return web.Response(text=html_content, content_type="text/html") - return web.Response(status=404) + async def _stop_server(self) -> None: + if self._server is not None: + self._server.close() + await self._server.wait_closed() + self._server = None async def run_servers(self, port: int, timeout: int) -> dict[str, Any]: - """Run both HTTP and WebSocket servers.""" + """Wait for the page to finish its checks, or for the timeout.""" import asyncio - from aiohttp import web - - # Create aiohttp app - app = web.Application() - app.router.add_get("/", self.http_handler) - app.router.add_get("/ws", self.websocket_handler) - - # Start HTTP server (which also handles WebSocket) - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, "localhost", port) - await site.start() - console.print( f"\n[green]✓[/green] Debug server started on port [cyan]{port}[/cyan]" ) @@ -233,9 +414,6 @@ async def run_servers(self, port: int, timeout: int) -> dict[str, Any]: while not self.debug_complete and (time.time() - start_time) < timeout: await asyncio.sleep(0.1) - # Cleanup - await runner.cleanup() - return { "completed": self.debug_complete, "timed_out": not self.debug_complete @@ -274,7 +452,7 @@ def display_results( log_table.add_row( time_str, f"[{type_style}]{msg.type.value}[/{type_style}]", - msg.message or "", + _untrusted(msg.message or ""), ) console.print(log_table) @@ -314,14 +492,21 @@ def display_results( status_color = "green" if cap.supported else "red" cap_table.add_row( display_name, - f"[{status_color}]{status_icon} {cap.details}[/{status_color}]", - cap.version or "", + Text.assemble( + f"{status_icon} ", + _untrusted(str(cap.details)), + style=status_color, + ), + _untrusted(cap.version or ""), ) console.print(cap_table) # Overall compatibility - console.print(f"\n[bold]User Agent:[/bold] {capabilities.user_agent}") + console.print( + Text("\nUser Agent:", style="bold"), + _untrusted(capabilities.user_agent), + ) if capabilities.overall_compatible: console.print( "[bold green]✓ Browser is fully compatible with " @@ -358,6 +543,7 @@ def handle( ) -> Any: """Handle browser debug command execution.""" import asyncio + import secrets port = kwargs.get("port") no_browser = kwargs.get("no_browser", False) @@ -365,10 +551,7 @@ def handle( save_report = kwargs.get("save_report") self.start_time = time.time() - - # Find available port - if not port: - port = self.find_available_port() + self._token = secrets.token_urlsafe(16) # Show initial message console.print( @@ -380,7 +563,23 @@ def handle( ) ) - url = f"http://localhost:{port}" + # Listen first, so the URL shown is the one that is being served. + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + port = loop.run_until_complete(self._start_server(port or None)) + result = self._serve(loop, port, timeout, no_browser) + finally: + loop.run_until_complete(self._stop_server()) + loop.close() + + return self._finish(port, timeout, save_report, result) + + def _serve( + self, loop: Any, port: int, timeout: int, no_browser: bool + ) -> dict[str, Any]: + """Point the browser at the page, then wait for its checks.""" + url = f"http://127.0.0.1:{port}/?t={self._token}" # Prompt user to open browser if not no_browser: @@ -389,18 +588,18 @@ def handle( "\n[yellow]Press Enter to open the debugger in your browser...[/yellow]" ) input() - else: - console.print( - f"\n[dim]Opening [cyan]{url}[/cyan] in your browser...[/dim]" - ) + _print_link( + "Opening the debugger in your browser:", + url, + f"If it doesn't open, paste the link into your browser. {LINK_NOTE}", + ) webbrowser.open(url) - browser_opened = True + self.browser_opened = True else: - console.print( - f"\n[dim]Open [cyan]{url}[/cyan] in your browser to start " - f"debugging[/dim]" + _print_link( + "Open this link in your browser to start debugging:", url, LINK_NOTE ) - browser_opened = False + self.browser_opened = False # Run the async event loop with Progress( @@ -410,18 +609,20 @@ def handle( transient=True, ) as progress: task = progress.add_task("Waiting for browser connection...", total=None) - - # Run servers - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - try: - result = loop.run_until_complete(self.run_servers(port, timeout)) - finally: - loop.close() - + result: dict[str, Any] = loop.run_until_complete( + self.run_servers(port, timeout) + ) progress.update(task, completed=True) + return result + def _finish( + self, + port: int, + timeout: int, + save_report: str | None, + result: dict[str, Any], + ) -> BrowserDebugResult: + """Show the results and build the command result.""" # Process results capabilities = None if self.capabilities_data and "overall_compatible" in self.capabilities_data: @@ -462,6 +663,22 @@ def handle( console.print("[dim]No browser connection was established.[/dim]") else: console.print("\n[yellow]Debug session ended[/yellow]") + left_out = [] + if self.dropped_messages: + left_out.append( + _count(self.dropped_messages, "message", "messages") + + " over the size limit" + ) + if self.dropped_capabilities: + left_out.append( + _count( + self.dropped_capabilities, + "unknown capability name", + "unknown capability names", + ) + ) + if left_out: + console.print(f"[dim]Left out of the results: {', '.join(left_out)}[/dim]") # Build result errors = [ @@ -483,7 +700,7 @@ def handle( errors=errors, warnings=warnings, duration_seconds=result["duration"], - browser_opened=browser_opened, + browser_opened=self.browser_opened, ) # Save report if requested diff --git a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html index b5bf50f0..12b7ece1 100644 --- a/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html +++ b/packages/deepctl-cmd-debug-browser/src/deepctl_cmd_debug_browser/static/debug.html @@ -232,8 +232,8 @@

Debug Complete!