diff --git a/plugins/dojo-verify/tests/_helpers.py b/plugins/dojo-verify/tests/_helpers.py index 4b09e6f..73639d9 100644 --- a/plugins/dojo-verify/tests/_helpers.py +++ b/plugins/dojo-verify/tests/_helpers.py @@ -9,6 +9,12 @@ import subprocess import sys import tempfile +import time + +try: + import resource +except ImportError: # Windows has no resource module + resource = None sys.dont_write_bytecode = True @@ -25,6 +31,20 @@ def uid(): return "u%06d" % _counter[0] +def cpu_seconds(): + """CPU time used so far by this process and its finished children (the hook runs as a waited subprocess). + + The speed tests guard against work that grows with the input. Wall-clock time also counts time spent waiting + for a CPU, so a busy machine failed them with no change to the code. Windows has no child CPU accounting here, + so it falls back to the wall clock. + """ + if resource is None: + return time.perf_counter() + me = resource.getrusage(resource.RUSAGE_SELF) + kids = resource.getrusage(resource.RUSAGE_CHILDREN) + return me.ru_utime + me.ru_stime + kids.ru_utime + kids.ru_stime + + def load_hook_module(): spec = importlib.util.spec_from_file_location("stop_evidence_under_test", HOOK) mod = importlib.util.module_from_spec(spec) diff --git a/plugins/dojo-verify/tests/test_stop_evidence.py b/plugins/dojo-verify/tests/test_stop_evidence.py index 006af79..d66190c 100644 --- a/plugins/dojo-verify/tests/test_stop_evidence.py +++ b/plugins/dojo-verify/tests/test_stop_evidence.py @@ -1,7 +1,6 @@ """The Stop hook, run as a subprocess against hand-written transcripts and payloads.""" import json import os -import time import unittest import _helpers as h @@ -801,9 +800,9 @@ def test_large_transcript_with_the_edit_and_check_inside_the_tail(self): + h.passing_check("b1") + [h.assistant_text("Fixed.")]) self.sb.write(entries) self.assertGreater(os.path.getsize(self.sb.transcript), 2 * 1024 * 1024) - start = time.time() + start = h.cpu_seconds() r = self.run_hook() - elapsed = time.time() - start + elapsed = h.cpu_seconds() - start self.assertTrue(r.silent) self.assertLess(elapsed, 1.5) @@ -828,10 +827,10 @@ def test_a_single_giant_line_is_skipped(self): def test_typical_case_is_fast(self): self.sb.write(self.claim_turn()) - start = time.time() + start = h.cpu_seconds() r = self.run_hook() self.assertTrue(r.blocked) - self.assertLess(time.time() - start, 1.0) + self.assertLess(h.cpu_seconds() - start, 1.0) class TestShellCompounds(Base): @@ -997,9 +996,9 @@ def test_pathological_messages_stay_fast_in_process(self): } for label, text in messages.items(): with self.subTest(label=label): - start = time.time() + start = h.cpu_seconds() hook.detect_claim(text, True) - self.assertLess(time.time() - start, 0.2) + self.assertLess(h.cpu_seconds() - start, 0.2) def test_questions_and_histories_stay_fast_in_process(self): hook = h.load_hook_module() @@ -1016,9 +1015,9 @@ def test_questions_and_histories_stay_fast_in_process(self): } for label, text in messages.items(): with self.subTest(label=label): - start = time.time() + start = h.cpu_seconds() hook.detect_claim(text, True) - self.assertLess(time.time() - start, 0.2) + self.assertLess(h.cpu_seconds() - start, 0.2) def test_a_command_with_thousands_of_branches_stays_fast_in_process(self): hook = h.load_hook_module() @@ -1028,17 +1027,17 @@ def test_a_command_with_thousands_of_branches_stays_fast_in_process(self): "never closed": "if a; then pytest -q; else echo no; " * 3000, }.items(): with self.subTest(label=label): - start = time.time() + start = h.cpu_seconds() hook.command_events(command) - self.assertLess(time.time() - start, 1.0) + self.assertLess(h.cpu_seconds() - start, 1.0) def test_a_message_that_fills_the_whole_input_still_finishes(self): self.sb.write(self.claim_turn()) text = "Fixed " + " " * (1024 * 1024 - 20000) - start = time.time() + start = h.cpu_seconds() r = self.run_hook(last_assistant_message=text) self.assertEqual((r.code, r.err), (0, "")) - self.assertLess(time.time() - start, 1.5) + self.assertLess(h.cpu_seconds() - start, 1.5) def test_a_claim_at_the_start_or_the_end_of_a_long_message_is_still_seen(self): hook = h.load_hook_module() diff --git a/scripts/release-check.sh b/scripts/release-check.sh index e94bde8..d77deba 100755 --- a/scripts/release-check.sh +++ b/scripts/release-check.sh @@ -312,25 +312,50 @@ step_mod() { # A mod's module is register.ts or register.tsx (a mod that draws JSX); either one is a mod to test. if [ ! -f "$dir/hooks/register.ts" ] && [ ! -f "$dir/hooks/register.tsx" ]; then add_row "mod:$p" SKIP "no hooks/register.ts(x)"; return 0; fi if [ -n "$CLAUDE_REASON" ]; then add_row "mod:$p" ERROR "$CLAUDE_REASON"; return 0; fi - local log="$TMP/mod-$p.log" - ( cd "$REPO" && env CLAUDE_CODE_ENABLE_FUNCTION_HOOKS=1 "$CLAUDE_BIN" plugin test "$dir" ) >"$log" 2>&1 - RC=$? + local log="$TMP/mod-$p.log" note="" + run_mod "$dir" "$log" + # The kit gives every test a fixed 5000 ms that a mod can't raise, and a loaded machine stalls tests past it + # (seen 2026-11-02 at load ~550: dojo-verify failed, then passed 109/109 in 4s on the same code). When every + # failure is that timeout, run once more and say so; a test that really hangs times out again and stays FAIL. + if [ "$RC" -ne 0 ] && only_kit_timeouts "$log"; then + note="rerun: the first run's $(grep -c '^(fail)' "$log") failure(s) were all kit timeouts, load $(load_avg)" + run_mod "$dir" "$log" + fi if grep -q 'hooks modules are turned off' "$log" 2>/dev/null; then add_row "mod:$p" ERROR "function hooks are off in this process" return 0 fi parse_ran "$log" if [ "$RC" -ne 0 ]; then - add_row "mod:$p" FAIL "exit $RC: $(tail_of "$log")" + add_row "mod:$p" FAIL "exit $RC: $(tail_of "$log")${note:+ ($note)}" elif [ -z "$RAN" ] || [ "$RAN" -eq 0 ]; then - add_row "mod:$p" FAIL "no tests ran (an empty run is not a pass)" + add_row "mod:$p" FAIL "no tests ran (an empty run is not a pass)${note:+ ($note)}" elif ! grep -Eq '^[[:space:]]*0 fail' "$log" 2>/dev/null; then - add_row "mod:$p" FAIL "output has no '0 fail' line" + add_row "mod:$p" FAIL "output has no '0 fail' line${note:+ ($note)}" + elif [ -n "$note" ]; then + add_row "mod:$p" WARN "$RAN tests on a $note" else add_row "mod:$p" PASS "$RAN tests" fi } +run_mod() { + ( cd "$REPO" && env CLAUDE_CODE_ENABLE_FUNCTION_HOOKS=1 "$CLAUDE_BIN" plugin test "$1" ) >"$2" 2>&1 + RC=$? +} + +# only_kit_timeouts LOG: true when the run failed tests and every failure is the kit's per-test timeout. +only_kit_timeouts() { + local fails timeouts + fails=$(grep -c '^(fail)' "$1" 2>/dev/null) || fails=0 + timeouts=$(grep -Ec 'timed out after [0-9]+ ms' "$1" 2>/dev/null) || timeouts=0 + [ "$fails" -gt 0 ] && [ "$fails" -eq "$timeouts" ] +} + +load_avg() { + uptime 2>/dev/null | sed -E 's/.*load averages?: *//' +} + step_self_test() { want self-test || return 0 if [ "$PY39_OK" != 1 ]; then add_row self-test ERROR "python 3.9 interpreter not found (set DOJO_PY39)"; return 0; fi diff --git a/scripts/tests_suite/test_release_check.py b/scripts/tests_suite/test_release_check.py index 839362b..e834db6 100644 --- a/scripts/tests_suite/test_release_check.py +++ b/scripts/tests_suite/test_release_check.py @@ -57,6 +57,13 @@ vacuous) printf ' 0 pass\\n 0 fail\\nRan 0 tests across 0 files.\\n'; exit 0 ;; noran) printf 'all good\\n'; exit 0 ;; fail) printf ' 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;; + timeout) printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;; + timeoutonce) + if [ "$(grep -c '^fh:' "$STUB_LOG")" -le 1 ]; then + printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1 + fi + printf ' 3 pass\\n 0 fail\\nRan 3 tests across 1 file.\\n'; exit 0 ;; + mixed) printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n(fail) wrong [3.00ms]\\n error: expect(received).toBe(expected)\\n 1 pass\\n 2 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;; off) echo "hooks modules are turned off in this process"; exit 1 ;; spam3) spam; exit 3 ;; esac @@ -216,6 +223,29 @@ def test_failing_mod_test(self): self.assertNotEqual(rc, 0) self.assertRow(rows, "mod:dojo-alpha", "FAIL") + def test_a_mod_run_that_only_timed_out_is_rerun_once_and_shown_as_warn(self): + # Seen 2026-11-02: at load ~550 dojo-verify's mod tests passed the kit's fixed 5000 ms timeout and the row + # failed; the same code passed 109/109 in 4s once the machine was quiet. + self.add_plugin("dojo-alpha", register=True) + rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="timeoutonce") + self.assertEqual(rc, 0, out + err) + self.assertRow(rows, "mod:dojo-alpha", "WARN", "kit timeouts") + self.assertEqual(self.logtext().count("fh:1"), 2) + + def test_a_mod_test_that_times_out_every_time_still_fails(self): + self.add_plugin("dojo-alpha", register=True) + rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="timeout") + self.assertNotEqual(rc, 0) + self.assertRow(rows, "mod:dojo-alpha", "FAIL", "kit timeouts") + self.assertEqual(self.logtext().count("fh:1"), 2) + + def test_a_real_failure_beside_a_timeout_is_not_rerun(self): + self.add_plugin("dojo-alpha", register=True) + rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="mixed") + self.assertNotEqual(rc, 0) + self.assertRow(rows, "mod:dojo-alpha", "FAIL") + self.assertEqual(self.logtext().count("fh:1"), 1) + def test_a_tsx_module_is_a_mod_and_is_tested_not_skipped(self): # Seen 2026-10-15: the step looked only for register.ts, so two mods written as register.tsx were SKIP # and the gate passed without running their tests.