Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions plugins/dojo-verify/tests/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@
import subprocess
import sys
import tempfile
import time

try:
import resource
except ImportError: # Windows has no resource module
resource = None

sys.dont_write_bytecode = True

Expand All @@ -25,6 +31,20 @@ def uid():
return "u%06d" % _counter[0]


def cpu_seconds():
"""CPU time used so far by this process and its finished children (the hook runs as a waited subprocess).

The speed tests guard against work that grows with the input. Wall-clock time also counts time spent waiting
for a CPU, so a busy machine failed them with no change to the code. Windows has no child CPU accounting here,
so it falls back to the wall clock.
"""
if resource is None:
return time.perf_counter()
me = resource.getrusage(resource.RUSAGE_SELF)
kids = resource.getrusage(resource.RUSAGE_CHILDREN)
return me.ru_utime + me.ru_stime + kids.ru_utime + kids.ru_stime


def load_hook_module():
spec = importlib.util.spec_from_file_location("stop_evidence_under_test", HOOK)
mod = importlib.util.module_from_spec(spec)
Expand Down
25 changes: 12 additions & 13 deletions plugins/dojo-verify/tests/test_stop_evidence.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""The Stop hook, run as a subprocess against hand-written transcripts and payloads."""
import json
import os
import time
import unittest

import _helpers as h
Expand Down Expand Up @@ -801,9 +800,9 @@ def test_large_transcript_with_the_edit_and_check_inside_the_tail(self):
+ h.passing_check("b1") + [h.assistant_text("Fixed.")])
self.sb.write(entries)
self.assertGreater(os.path.getsize(self.sb.transcript), 2 * 1024 * 1024)
start = time.time()
start = h.cpu_seconds()
r = self.run_hook()
elapsed = time.time() - start
elapsed = h.cpu_seconds() - start
self.assertTrue(r.silent)
self.assertLess(elapsed, 1.5)

Expand All @@ -828,10 +827,10 @@ def test_a_single_giant_line_is_skipped(self):

def test_typical_case_is_fast(self):
self.sb.write(self.claim_turn())
start = time.time()
start = h.cpu_seconds()
r = self.run_hook()
self.assertTrue(r.blocked)
self.assertLess(time.time() - start, 1.0)
self.assertLess(h.cpu_seconds() - start, 1.0)


class TestShellCompounds(Base):
Expand Down Expand Up @@ -997,9 +996,9 @@ def test_pathological_messages_stay_fast_in_process(self):
}
for label, text in messages.items():
with self.subTest(label=label):
start = time.time()
start = h.cpu_seconds()
hook.detect_claim(text, True)
self.assertLess(time.time() - start, 0.2)
self.assertLess(h.cpu_seconds() - start, 0.2)

def test_questions_and_histories_stay_fast_in_process(self):
hook = h.load_hook_module()
Expand All @@ -1016,9 +1015,9 @@ def test_questions_and_histories_stay_fast_in_process(self):
}
for label, text in messages.items():
with self.subTest(label=label):
start = time.time()
start = h.cpu_seconds()
hook.detect_claim(text, True)
self.assertLess(time.time() - start, 0.2)
self.assertLess(h.cpu_seconds() - start, 0.2)

def test_a_command_with_thousands_of_branches_stays_fast_in_process(self):
hook = h.load_hook_module()
Expand All @@ -1028,17 +1027,17 @@ def test_a_command_with_thousands_of_branches_stays_fast_in_process(self):
"never closed": "if a; then pytest -q; else echo no; " * 3000,
}.items():
with self.subTest(label=label):
start = time.time()
start = h.cpu_seconds()
hook.command_events(command)
self.assertLess(time.time() - start, 1.0)
self.assertLess(h.cpu_seconds() - start, 1.0)

def test_a_message_that_fills_the_whole_input_still_finishes(self):
self.sb.write(self.claim_turn())
text = "Fixed " + " " * (1024 * 1024 - 20000)
start = time.time()
start = h.cpu_seconds()
r = self.run_hook(last_assistant_message=text)
self.assertEqual((r.code, r.err), (0, ""))
self.assertLess(time.time() - start, 1.5)
self.assertLess(h.cpu_seconds() - start, 1.5)

def test_a_claim_at_the_start_or_the_end_of_a_long_message_is_still_seen(self):
hook = h.load_hook_module()
Expand Down
37 changes: 31 additions & 6 deletions scripts/release-check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -312,25 +312,50 @@ step_mod() {
# A mod's module is register.ts or register.tsx (a mod that draws JSX); either one is a mod to test.
if [ ! -f "$dir/hooks/register.ts" ] && [ ! -f "$dir/hooks/register.tsx" ]; then add_row "mod:$p" SKIP "no hooks/register.ts(x)"; return 0; fi
if [ -n "$CLAUDE_REASON" ]; then add_row "mod:$p" ERROR "$CLAUDE_REASON"; return 0; fi
local log="$TMP/mod-$p.log"
( cd "$REPO" && env CLAUDE_CODE_ENABLE_FUNCTION_HOOKS=1 "$CLAUDE_BIN" plugin test "$dir" ) >"$log" 2>&1
RC=$?
local log="$TMP/mod-$p.log" note=""
run_mod "$dir" "$log"
# The kit gives every test a fixed 5000 ms that a mod can't raise, and a loaded machine stalls tests past it
# (seen 2026-11-02 at load ~550: dojo-verify failed, then passed 109/109 in 4s on the same code). When every
# failure is that timeout, run once more and say so; a test that really hangs times out again and stays FAIL.
if [ "$RC" -ne 0 ] && only_kit_timeouts "$log"; then
note="rerun: the first run's $(grep -c '^(fail)' "$log") failure(s) were all kit timeouts, load $(load_avg)"
run_mod "$dir" "$log"
fi
if grep -q 'hooks modules are turned off' "$log" 2>/dev/null; then
add_row "mod:$p" ERROR "function hooks are off in this process"
return 0
fi
parse_ran "$log"
if [ "$RC" -ne 0 ]; then
add_row "mod:$p" FAIL "exit $RC: $(tail_of "$log")"
add_row "mod:$p" FAIL "exit $RC: $(tail_of "$log")${note:+ ($note)}"
elif [ -z "$RAN" ] || [ "$RAN" -eq 0 ]; then
add_row "mod:$p" FAIL "no tests ran (an empty run is not a pass)"
add_row "mod:$p" FAIL "no tests ran (an empty run is not a pass)${note:+ ($note)}"
elif ! grep -Eq '^[[:space:]]*0 fail' "$log" 2>/dev/null; then
add_row "mod:$p" FAIL "output has no '0 fail' line"
add_row "mod:$p" FAIL "output has no '0 fail' line${note:+ ($note)}"
elif [ -n "$note" ]; then
add_row "mod:$p" WARN "$RAN tests on a $note"
else
add_row "mod:$p" PASS "$RAN tests"
fi
}

run_mod() {
( cd "$REPO" && env CLAUDE_CODE_ENABLE_FUNCTION_HOOKS=1 "$CLAUDE_BIN" plugin test "$1" ) >"$2" 2>&1
RC=$?
}

# only_kit_timeouts LOG: true when the run failed tests and every failure is the kit's per-test timeout.
only_kit_timeouts() {
local fails timeouts
fails=$(grep -c '^(fail)' "$1" 2>/dev/null) || fails=0
timeouts=$(grep -Ec 'timed out after [0-9]+ ms' "$1" 2>/dev/null) || timeouts=0
[ "$fails" -gt 0 ] && [ "$fails" -eq "$timeouts" ]
}

load_avg() {
uptime 2>/dev/null | sed -E 's/.*load averages?: *//'
}

step_self_test() {
want self-test || return 0
if [ "$PY39_OK" != 1 ]; then add_row self-test ERROR "python 3.9 interpreter not found (set DOJO_PY39)"; return 0; fi
Expand Down
30 changes: 30 additions & 0 deletions scripts/tests_suite/test_release_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,13 @@
vacuous) printf ' 0 pass\\n 0 fail\\nRan 0 tests across 0 files.\\n'; exit 0 ;;
noran) printf 'all good\\n'; exit 0 ;;
fail) printf ' 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;;
timeout) printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;;
timeoutonce)
if [ "$(grep -c '^fh:' "$STUB_LOG")" -le 1 ]; then
printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n 2 pass\\n 1 fail\\nRan 3 tests across 1 file.\\n'; exit 1
fi
printf ' 3 pass\\n 0 fail\\nRan 3 tests across 1 file.\\n'; exit 0 ;;
mixed) printf '(fail) slow [5001.00ms]\\n Error: timed out after 5000 ms\\n(fail) wrong [3.00ms]\\n error: expect(received).toBe(expected)\\n 1 pass\\n 2 fail\\nRan 3 tests across 1 file.\\n'; exit 1 ;;
off) echo "hooks modules are turned off in this process"; exit 1 ;;
spam3) spam; exit 3 ;;
esac
Expand Down Expand Up @@ -216,6 +223,29 @@ def test_failing_mod_test(self):
self.assertNotEqual(rc, 0)
self.assertRow(rows, "mod:dojo-alpha", "FAIL")

def test_a_mod_run_that_only_timed_out_is_rerun_once_and_shown_as_warn(self):
# Seen 2026-11-02: at load ~550 dojo-verify's mod tests passed the kit's fixed 5000 ms timeout and the row
# failed; the same code passed 109/109 in 4s once the machine was quiet.
self.add_plugin("dojo-alpha", register=True)
rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="timeoutonce")
self.assertEqual(rc, 0, out + err)
self.assertRow(rows, "mod:dojo-alpha", "WARN", "kit timeouts")
self.assertEqual(self.logtext().count("fh:1"), 2)

def test_a_mod_test_that_times_out_every_time_still_fails(self):
self.add_plugin("dojo-alpha", register=True)
rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="timeout")
self.assertNotEqual(rc, 0)
self.assertRow(rows, "mod:dojo-alpha", "FAIL", "kit timeouts")
self.assertEqual(self.logtext().count("fh:1"), 2)

def test_a_real_failure_beside_a_timeout_is_not_rerun(self):
self.add_plugin("dojo-alpha", register=True)
rc, out, err, rows = self.run_rc("--plugin", "dojo-alpha", STUB_MOD_MODE="mixed")
self.assertNotEqual(rc, 0)
self.assertRow(rows, "mod:dojo-alpha", "FAIL")
self.assertEqual(self.logtext().count("fh:1"), 1)

def test_a_tsx_module_is_a_mod_and_is_tested_not_skipped(self):
# Seen 2026-10-15: the step looked only for register.ts, so two mods written as register.tsx were SKIP
# and the gate passed without running their tests.
Expand Down
Loading