From 7175889b2c8c53313435c70731ed4fc90ddb6ae9 Mon Sep 17 00:00:00 2001 From: Lyra Date: Mon, 2 Nov 2026 10:36:07 +0000 Subject: [PATCH 1/3] feat: post the visual-diff shots in one marked comment from a trusted poster (#74) visual-diff.yml runs with a read-only token, a fork's pull request's included, so it can only keep its shots as an artifact. visual-diff-comment.yml runs scripts/visual-diff-post.mjs from main after "Visual diff" completes: it reads only the expected PNGs from the untrusted artifact (signature, size cap, exact dimensions, no links), posts only on the open pull request whose head commit, repo and branch are the run's, commits the shots to the orphan branch visual-diff-shots under pr-//, and creates or rewrites one comment marked . The only ref it writes is refs/heads/visual-diff-shots, pinned by test/visual-diff-post.test.mjs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/visual-diff-comment.yml | 55 +++++ AGENTS.md | 7 + scripts/visual-diff-post.mjs | 239 ++++++++++++++++++ test/visual-diff-post.test.mjs | 286 ++++++++++++++++++++++ 4 files changed, 587 insertions(+) create mode 100644 .github/workflows/visual-diff-comment.yml create mode 100644 scripts/visual-diff-post.mjs create mode 100644 test/visual-diff-post.test.mjs diff --git a/.github/workflows/visual-diff-comment.yml b/.github/workflows/visual-diff-comment.yml new file mode 100644 index 0000000..5980390 --- /dev/null +++ b/.github/workflows/visual-diff-comment.yml @@ -0,0 +1,55 @@ +name: Visual diff comment + +# setkernel-live#74, slice 2: posts the shots that "Visual diff" (visual-diff.yml) kept, in one +# comment on the pull request marked . That run has a read-only token, +# a fork's included, so it can't post; this one runs from main after it completes, with this repo's +# token. It never checks out or runs the pull request's code: scripts/visual-diff-post.mjs (main's) +# treats the artifact as untrusted, uses the pull request only if its head is the run's head, and +# writes only to the visual-diff-shots branch. Not a required check: visual-diff is. +on: + workflow_run: + workflows: [Visual diff] + types: [completed] + +permissions: {} + +# A newer run for the same branch supersedes this one; pushes from different pull requests to +# visual-diff-shots retry on their own (publish in the script). +concurrency: + group: visual-diff-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + +jobs: + # Public repo: what a fork's pull request made is handled here, so never a self-hosted runner. + post: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read # the run's artifact + contents: write # visual-diff-shots, and only that ref (scripts/visual-diff-post.mjs) + pull-requests: write # the comment + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + # No artifact when no UI path changed: nothing to post. + - id: find + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/visual-diff-post.mjs find >> "$GITHUB_OUTPUT" + - if: steps.find.outputs.artifact == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: visual-diff + path: ${{ runner.temp }}/visual-diff + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - if: steps.find.outputs.artifact == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/visual-diff-post.mjs post "$RUNNER_TEMP/visual-diff" >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 7ff24f7..27354f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ Public repo: everything here is visible to anyone. # headline/demo/replay, Day's lanes/Now column/key/scrubbing/fitWall, # Buzz alert, page hosts, fonts and contrast, the smoke check, the showcase's # publisher, the workflow runner guard, the visual-diff check's plan and workflow, +# its poster (the artifact, the pull request, the shots branch, the comment), # and repo names against GitHub's public list, live) # worker: npm run test:worker (vitest in workerd: the real Worker and Hub, see test/worker/) # lint: npm run check (node --check on every script) @@ -69,6 +70,12 @@ a laptop. from the head at 1440x900 and 375x812, on one runner with a pinned Playwright, and keeps the shots as the run's `visual-diff` artifact. It fails only when the head page errs. It always runs, and passes with no UI change, so it can be a required check: never give it a `paths:` filter or a job `if:`. +- `scripts/visual-diff-post.mjs` and `.github/workflows/visual-diff-comment.yml`: its poster. A + `workflow_run` after "Visual diff", from main, with the only write token a pull request's run can + reach: it commits the shots to the orphan branch `visual-diff-shots` (`pr-//`) and keeps + one comment marked ``. The artifact is untrusted (a fork's pull request + can change the workflow that made it): never check out or run the pull request's code there, never + put text from the artifact in the comment, and never write any ref but `visual-diff-shots`. - `test/`: `node --test` suites for the pure modules and the page (`day.test.mjs` for Day). - `test/worker/`: vitest suites run in workerd by `@cloudflare/vitest-plugin` (`vitest.config.mjs` reads `wrangler.jsonc`). They fake Access's certs with a throwaway key and drive every route diff --git a/scripts/visual-diff-post.mjs b/scripts/visual-diff-post.mjs new file mode 100644 index 0000000..dbe403d --- /dev/null +++ b/scripts/visual-diff-post.mjs @@ -0,0 +1,239 @@ +// The visual-diff check's poster (setkernel-live#74, slice 2). .github/workflows/visual-diff.yml runs +// on a pull request with a read-only token, a fork's included, so it can only keep its shots as the +// run's visual-diff artifact. .github/workflows/visual-diff-comment.yml runs this, from main, when +// that run completes: it commits the shots to the visual-diff-shots branch, under +// pr-//, and posts them in one comment on the pull request, marked +// . A later head rewrites the same comment. +// +// node scripts/visual-diff-post.mjs find prints artifact=true or artifact=false +// node scripts/visual-diff-post.mjs post publishes , the downloaded artifact; prints a summary +// +// Both read the workflow_run event from GITHUB_EVENT_PATH, and GITHUB_REPOSITORY and GITHUB_TOKEN. +// +// The artifact is untrusted: a pull request, a fork's too, can change the workflow that made it. So +// only the expected PNGs are read, each checked for its signature, size and dimensions; nothing in +// it is run, and no text from it reaches the comment. Its pull request number is only a candidate: +// the pull request is used only if it is open here and its head is the run's head (the event's +// head_sha). The only ref this ever writes is refs/heads/visual-diff-shots. + +import { lstat, readFile, readdir } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { SHOTS } from "./visual-diff.mjs"; + +export const SHOTS_BRANCH = "visual-diff-shots"; +const SHOTS_REF = `refs/heads/${SHOTS_BRANCH}`; +export const MAX_PNG = 5 * 1024 * 1024; +const MAX_JSON = 64 * 1024; +const SHA = /^[0-9a-f]{40}$/; +const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const BOT = "github-actions[bot]"; + +export const marker = (sha) => ``; +const MARKED = //; + +// A PNG of exactly width x height: its signature, then IHDR, which holds the size. +export function checkPng(buf, width, height) { + if (buf.length > MAX_PNG) return `over ${MAX_PNG} bytes`; + if (buf.length < 24 || !buf.subarray(0, 8).equals(SIGNATURE) || buf.toString("latin1", 12, 16) !== "IHDR") + return "not a PNG"; + const [w, h] = [buf.readUInt32BE(16), buf.readUInt32BE(20)]; + return w === width && h === height ? null : `${w} x ${h}, not ${width} x ${height}`; +} + +// The downloaded artifact: base/.png, head/.png and visual-diff.json, nothing else, no +// links. A shot that failed to draw has no PNG. Throws on anything unexpected. +export async function readArtifact(dir) { + const allowed = new Set(["visual-diff.json", "base", "head", + ...SHOTS.flatMap(([name]) => [`base/${name}.png`, `head/${name}.png`])]); + const walk = async (rel) => { + for (const name of await readdir(join(dir, rel))) { + const path = rel ? `${rel}/${name}` : name; + if (!allowed.has(path)) throw new Error(`unexpected file in the artifact: ${JSON.stringify(path)}`); + const st = await lstat(join(dir, path)); + const folder = path === "base" || path === "head"; + if (folder ? !st.isDirectory() : !st.isFile()) throw new Error(`not a plain ${folder ? "folder" : "file"}: ${path}`); + if (folder) await walk(path); + } + }; + await walk(""); + const read = async (path, max) => { + const st = await lstat(join(dir, path)).catch(() => null); + if (!st) return null; + if (st.size > max) throw new Error(`${path} is over ${max} bytes`); + return readFile(join(dir, path)); + }; + const shots = []; + for (const [name, width, height] of SHOTS) { + const shot = { name, width, height }; + for (const side of ["base", "head"]) { + const png = await read(`${side}/${name}.png`, MAX_PNG); + const bad = png && checkPng(png, width, height); + if (bad) throw new Error(`${side}/${name}.png: ${bad}`); + shot[side] = png; + } + shot.changed = !!(shot.base && shot.head) && !shot.base.equals(shot.head); + shots.push(shot); + } + const json = await read("visual-diff.json", MAX_JSON); + let number = null; + try { + const n = JSON.parse(json)?.pr?.number; + if (Number.isSafeInteger(n) && n > 0) number = n; + } catch {} + return { number, shots }; +} + +// The REST API with the job's token. fetch is a parameter so the tests can stand in for GitHub. +export function github({ repo, token, fetch = globalThis.fetch, base = "https://api.github.com" }) { + return async (method, path, body) => { + const res = await fetch(`${base}/repos/${repo}${path}`, { + method, + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + ...(body ? { "Content-Type": "application/json" } : {}) }, + body: body ? JSON.stringify(body) : undefined, + }); + const data = res.status === 204 ? null : await res.json().catch(() => null); + return { status: res.status, data }; + }; +} + +const ok = (r, what) => { + if (r.status >= 200 && r.status < 300) return r.data; + throw new Error(`${what}: HTTP ${r.status}${r.data?.message ? ` (${r.data.message})` : ""}`); +}; + +// The open pull request here whose head is the run's: the same commit, from the same repo and +// branch, so another pull request on the same commit can't take its comment. Candidates come from +// the event and the artifact; each is checked against GitHub. null if none matches (closed, moved +// on, or a lie). +export async function findPull(api, repo, run) { + for (const n of new Set([...run.pulls, ...run.claimed].filter((n) => Number.isSafeInteger(n) && n > 0))) { + const r = await api("GET", `/pulls/${n}`); + if (r.status === 404) continue; + const pr = ok(r, `pull request ${n}`); + if (pr.state === "open" && pr.base?.repo?.full_name === repo && pr.head?.sha === run.headSha && + pr.head?.repo?.full_name === run.headRepo && pr.head?.ref === run.headBranch) return n; + } + return null; +} + +// Commits the shots under pr-// on top of visual-diff-shots (an orphan branch, made on the +// first run) and returns the commit that holds them. Another pull request's poster may move the +// branch meanwhile, so a rejected update starts again from the new tip. +export async function publish(api, number, headSha, shots, tries = 5) { + const files = shots.flatMap((s) => ["base", "head"].filter((side) => s[side]) + .map((side) => ({ path: `pr-${number}/${headSha}/${side}/${s.name}.png`, png: s[side] }))); + const tree = []; + for (const f of files) { + const blob = ok(await api("POST", "/git/blobs", { content: f.png.toString("base64"), encoding: "base64" }), "blob"); + tree.push({ path: f.path, mode: "100644", type: "blob", sha: blob.sha }); + } + for (let i = 0; i < tries; i++) { + const ref = await api("GET", `/git/ref/heads/${SHOTS_BRANCH}`); + const parent = ref.status === 404 ? null : ok(ref, SHOTS_BRANCH).object.sha; + const parentTree = parent && ok(await api("GET", `/git/commits/${parent}`), "commit").tree.sha; + const made = ok(await api("POST", "/git/trees", { ...(parentTree ? { base_tree: parentTree } : {}), tree }), "tree"); + if (made.sha === parentTree) return parent; // a re-run: these shots are already there + const commit = ok(await api("POST", "/git/commits", { + message: `visual-diff: pr-${number} ${headSha}`, tree: made.sha, parents: parent ? [parent] : [], + }), "commit").sha; + const moved = parent + ? await api("PATCH", `/git/refs/heads/${SHOTS_BRANCH}`, { sha: commit, force: false }) + : await api("POST", "/git/refs", { ref: SHOTS_REF, sha: commit }); + if (moved.status === 422 || moved.status === 409) continue; // not a fast-forward, or made meanwhile + ok(moved, `update ${SHOTS_BRANCH}`); + return commit; + } + throw new Error(`${SHOTS_BRANCH} kept moving; gave up after ${tries} tries`); +} + +const CONCLUSION = { + success: "The check passed.", + failure: "The check failed: the head didn't draw cleanly. The run's summary says why.", +}; + +// The comment, from this file and the event only: no text from the artifact. +export function commentBody({ repo, headSha, commit, number, runUrl, conclusion, shots }) { + const raw = (side, name) => `https://raw.githubusercontent.com/${repo}/${commit}/pr-${number}/${headSha}/${side}/${name}.png`; + const cell = (s, side) => s[side] ? `![${side}, ${s.width} x ${s.height}](${raw(side, s.name)})` : "not drawn"; + const state = (s) => !s.head ? "the head didn't draw" : !s.base ? "the base didn't draw" : s.changed ? "changed" : "same"; + const changed = shots.filter((s) => s.changed).length; + return [ + marker(headSha), + "### Visual diff", + "", + `\`?demo\` drawn from this pull request's base and from its head, \`${headSha.slice(0, 7)}\`, in the ` + + `[Visual diff run](${runUrl}). ${changed ? `${changed} of ${shots.length} shots changed.` : "No shot changed."}` + + (CONCLUSION[conclusion] ? ` ${CONCLUSION[conclusion]}` : ""), + ...shots.flatMap((s) => [ + "", + `#### ${s.width} x ${s.height}: ${state(s)}`, + "", + "| Base | Head |", + "| --- | --- |", + `| ${cell(s, "base")} | ${cell(s, "head")} |`, + ]), + "", + ].join("\n"); +} + +// The poster's own marked comment, whatever head it names. Only the bot's: anyone can paste a marker. +export async function findComment(api, number) { + for (let page = 1; page <= 30; page++) { + const list = ok(await api("GET", `/issues/${number}/comments?per_page=100&page=${page}`), "comments"); + const hit = list.find((c) => c.user?.login === BOT && MARKED.test(c.body ?? "")); + if (hit) return hit.id; + if (list.length < 100) return null; + } + return null; +} + +export async function comment(api, number, body) { + const id = await findComment(api, number); + if (id) ok(await api("PATCH", `/issues/comments/${id}`, { body }), "update comment"); + else ok(await api("POST", `/issues/${number}/comments`, { body }), "post comment"); + return id ? "updated" : "posted"; +} + +// The completed run, from the event GitHub wrote (not from the artifact). +export function runFromEvent(event) { + const run = event?.workflow_run; + if (!run || run.event !== "pull_request" || !SHA.test(run.head_sha ?? "")) return null; + return { id: run.id, headSha: run.head_sha, headRepo: run.head_repository?.full_name, headBranch: run.head_branch, + url: run.html_url, conclusion: run.conclusion, pulls: (run.pull_requests ?? []).map((p) => p.number) }; +} + +// find: did that run keep a visual-diff artifact? (None when no UI path changed.) +export async function find(api, run) { + if (!run) return false; + const r = ok(await api("GET", `/actions/runs/${run.id}/artifacts?name=visual-diff`), "artifacts"); + return (r.artifacts ?? []).some((a) => a.name === "visual-diff" && !a.expired); +} + +export async function post(api, repo, run, dir) { + if (!run) return "Not a pull request's run, so nothing was posted."; + const { number: claimed, shots } = await readArtifact(dir); + const number = await findPull(api, repo, { ...run, claimed: [claimed] }); + if (!number) return `No open pull request has head \`${run.headSha}\` any more, so nothing was posted.`; + if (!shots.some((s) => s.base || s.head)) return "The artifact holds no shots, so nothing was posted."; + const commit = await publish(api, number, run.headSha, shots); + const body = commentBody({ repo, headSha: run.headSha, commit, number, runUrl: run.url, conclusion: run.conclusion, shots }); + return `Shots for \`${run.headSha}\` are on ${SHOTS_BRANCH} at \`${commit}\`; ${await comment(api, number, body)} the comment on #${number}.`; +} + +// Not import.meta.main: that needs Node 24.2 (scripts/smoke.mjs says the same). +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [cmd, ...args] = process.argv.slice(2); + if ((cmd === "find" && !args.length) || (cmd === "post" && args.length === 1)) { + const repo = process.env.GITHUB_REPOSITORY; + const run = runFromEvent(JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"))); + const api = github({ repo, token: process.env.GITHUB_TOKEN }); + if (cmd === "find") console.log(`artifact=${await find(api, run)}`); + else console.log(`### Visual diff comment\n\n${await post(api, repo, run, args[0])}`); + } else { + console.error("usage: visual-diff-post.mjs find | visual-diff-post.mjs post "); + process.exitCode = 2; + } +} diff --git a/test/visual-diff-post.test.mjs b/test/visual-diff-post.test.mjs new file mode 100644 index 0000000..3837f9c --- /dev/null +++ b/test/visual-diff-post.test.mjs @@ -0,0 +1,286 @@ +// The visual-diff check's poster (setkernel-live#74, slice 2): it reads only the expected PNGs from +// the untrusted artifact, posts only on the open pull request whose head is the run's head, writes +// only refs/heads/visual-diff-shots, and keeps one marked comment; and its workflow's shape. GitHub +// is a fake here. Run: npm test +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { SHOTS } from "../scripts/visual-diff.mjs"; +import { + MAX_PNG, SHOTS_BRANCH, checkPng, commentBody, find, github, marker, post, readArtifact, runFromEvent, +} from "../scripts/visual-diff-post.mjs"; + +const REPO = "setkernel/setkernel-live"; +const HEAD = "a".repeat(40); +const OTHER = "b".repeat(40); +const [[WIDE, W, H], [NARROW, w, h]] = SHOTS; + +const png = (width, height, fill = 0) => { + const b = Buffer.alloc(64, fill); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(b); + b.writeUInt32BE(13, 8); + b.write("IHDR", 12, "latin1"); + b.writeUInt32BE(width, 16); + b.writeUInt32BE(height, 20); + return b; +}; + +// An artifact as visual-diff.yml uploads it: { "base/.png": Buffer, ..., "visual-diff.json": "..." }. +async function artifact(files) { + const dir = await mkdtemp(join(process.env.TMPDIR ?? tmpdir(), "vd-")); + for (const [path, body] of Object.entries(files)) { + if (body === undefined) continue; + await mkdir(join(dir, path, ".."), { recursive: true }); + await writeFile(join(dir, path), body); + } + return dir; +} +const good = (number = 7) => ({ + [`base/${WIDE}.png`]: png(W, H), [`head/${WIDE}.png`]: png(W, H, 1), + [`base/${NARROW}.png`]: png(w, h), [`head/${NARROW}.png`]: png(w, h), + "visual-diff.json": JSON.stringify({ pr: { number, head: HEAD }, shots: [] }), +}); + +// A fake GitHub: pulls by number, the shots branch, comments; records every request. +function fakeGitHub({ pulls = {}, branch = null, comments = [], moves = 0, artifacts = [] } = {}) { + const calls = []; + const state = { branch, comments: [...comments], n: 0, moves: 0 }; + const sha = () => (++state.n).toString(16).padStart(40, "0"); + const api = async (method, path, body) => { + calls.push({ method, path, body }); + const reply = (status, data) => ({ status, data }); + let m; + if (method === "GET" && (m = path.match(/^\/pulls\/(\d+)$/))) + return pulls[m[1]] ? reply(200, pulls[m[1]]) : reply(404, { message: "Not Found" }); + if (method === "GET" && path.startsWith("/actions/runs/")) return reply(200, { artifacts }); + if (method === "GET" && path === `/git/ref/heads/${SHOTS_BRANCH}`) + return state.branch ? reply(200, { object: { sha: state.branch } }) : reply(404, { message: "Not Found" }); + if (method === "GET" && (m = path.match(/^\/git\/commits\/(\w+)$/))) return reply(200, { tree: { sha: `tree-of-${m[1]}` } }); + if (method === "POST" && path === "/git/blobs") return reply(201, { sha: sha() }); + if (method === "POST" && path === "/git/trees") return reply(201, { sha: sha() }); + if (method === "POST" && path === "/git/commits") return reply(201, { sha: sha() }); + if (method === "POST" && path === "/git/refs") { + if (state.branch) return reply(422, { message: "Reference already exists" }); + state.branch = body.sha; + return reply(201, {}); + } + if (method === "PATCH" && path === `/git/refs/heads/${SHOTS_BRANCH}`) { + if (state.moves < moves) { state.moves++; state.branch = sha(); return reply(422, { message: "not a fast forward" }); } + state.branch = body.sha; + return reply(200, {}); + } + if (method === "GET" && (m = path.match(/^\/issues\/(\d+)\/comments\?/))) return reply(200, state.comments); + if (method === "POST" && (m = path.match(/^\/issues\/(\d+)\/comments$/))) { + state.comments.push({ id: 99, user: { login: "github-actions[bot]" }, body: body.body }); + return reply(201, {}); + } + if (method === "PATCH" && (m = path.match(/^\/issues\/comments\/(\d+)$/))) { + const c = state.comments.find((c) => c.id === Number(m[1])); + c.body = body.body; + return reply(200, {}); + } + return reply(500, { message: `unexpected ${method} ${path}` }); + }; + return { api, calls, state }; +} +const openPull = (sha = HEAD, head = {}) => + ({ state: "open", head: { sha, ref: "change", repo: { full_name: "someone/fork" }, ...head }, base: { repo: { full_name: REPO } } }); +const run = (over = {}) => ({ id: 42, headSha: HEAD, headRepo: "someone/fork", headBranch: "change", + url: "https://github.com/setkernel/setkernel-live/actions/runs/42", conclusion: "success", pulls: [], ...over }); +const writes = (calls) => calls.filter((c) => c.method !== "GET"); + +test("a PNG must have the shot's exact size and stay under the cap", () => { + assert.equal(checkPng(png(W, H), W, H), null); + assert.equal(checkPng(png(w, h), W, H), `${w} x ${h}, not ${W} x ${H}`); + assert.equal(checkPng(Buffer.from("".padEnd(64)), W, H), "not a PNG"); + assert.equal(checkPng(Buffer.alloc(10), W, H), "not a PNG"); + assert.equal(checkPng(Buffer.concat([png(W, H), Buffer.alloc(MAX_PNG)]), W, H), `over ${MAX_PNG} bytes`); +}); + +test("the artifact is read as the expected PNGs and nothing else", async (t) => { + const dirs = []; + t.after(() => Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true })))); + const make = async (files) => { const d = await artifact(files); dirs.push(d); return d; }; + + const { number, shots } = await readArtifact(await make(good())); + assert.equal(number, 7); + assert.deepEqual(shots.map((s) => [s.name, s.changed]), [[WIDE, true], [NARROW, false]]); + + // A shot that didn't draw has no PNG; a JSON that isn't one only loses the candidate number. + const { [`head/${NARROW}.png`]: _, ...missing } = good(); + const partial = await readArtifact(await make({ ...missing, "visual-diff.json": "{not json" })); + assert.equal(partial.number, null); + assert.equal(partial.shots[1].head, null); + assert.equal(partial.shots[1].changed, false); + for (const n of [0, -1, 1.5, "7", 2 ** 60]) { + const r = await readArtifact(await make({ ...good(), "visual-diff.json": JSON.stringify({ pr: { number: n } }) })); + assert.equal(r.number, null, String(n)); + } + + for (const [files, why] of [ + [{ ...good(), "base/run.sh": "echo" }, /unexpected file.*base\/run\.sh/], + [{ ...good(), "head/x/y.png": png(W, H) }, /unexpected file.*head\/x/], + [{ ...missing, "visual-diff.json": undefined, "visual-diff.json/a": "" }, /not a plain file: visual-diff\.json/], + [{ ...good(), [`head/${WIDE}.png`]: png(w, h) }, /head\/demo-1440x900\.png: 375 x 812, not 1440 x 900/], + [{ ...good(), [`base/${NARROW}.png`]: "not a png" }, /not a PNG/], + [{ ...good(), "visual-diff.json": "x".repeat(70_000) }, /over/], + ]) await assert.rejects(readArtifact(await make(files)), why); + + const linked = await make(good()); + await rm(join(linked, `head/${WIDE}.png`)); + await symlink("/etc/passwd", join(linked, `head/${WIDE}.png`)); + await assert.rejects(readArtifact(linked), /not a plain file/); +}); + +test("it posts only on the open pull request whose head is the run's head", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + // Gone, moved on, closed, into another repo, or the same commit from another fork or branch. + for (const pulls of [{}, { 7: openPull(OTHER) }, { 7: { ...openPull(), state: "closed" } }, + { 7: { ...openPull(), base: { repo: { full_name: "someone/fork" } } } }, + { 7: openPull(HEAD, { repo: { full_name: "victim/fork" } }) }, { 7: openPull(HEAD, { ref: "other" }) }, + { 7: openPull(HEAD, { repo: null }) }]) { + const gh = fakeGitHub({ pulls }); + assert.match(await post(gh.api, REPO, run(), dir), /nothing was posted/); + assert.deepEqual(writes(gh.calls), [], "no write when the pull request doesn't match"); + } + // The event's pull request is tried first, and a false claim in the artifact is skipped. + const gh = fakeGitHub({ pulls: { 7: openPull(OTHER), 8: openPull() } }); + assert.match(await post(gh.api, REPO, run({ pulls: [8] }), dir), /posted the comment on #8/); + assert.ok(writes(gh.calls).every((c) => !c.path.includes("/issues/7/"))); + // A fork's run has no pull request in the event: the artifact's candidate, once checked, is used. + const fork = fakeGitHub({ pulls: { 5: { ...openPull(), state: "closed" }, 7: openPull() } }); + assert.match(await post(fork.api, REPO, run({ pulls: [5] }), dir), /posted the comment on #7/); + assert.equal(await post(fakeGitHub().api, REPO, null, dir), "Not a pull request's run, so nothing was posted."); +}); + +test("the only ref it writes is refs/heads/visual-diff-shots, whatever the artifact says", async (t) => { + const evil = await artifact({ ...good(7), "visual-diff.json": JSON.stringify({ pr: { number: 7, head: "main", ref: "refs/heads/main" } }) }); + t.after(() => rm(evil, { recursive: true, force: true })); + // First run: the branch is made as an orphan. Later: a fast-forward, retried when another poster moved it. + for (const opts of [{}, { branch: "c".repeat(40) }, { branch: "c".repeat(40), moves: 2 }]) { + const gh = fakeGitHub({ pulls: { 7: openPull() }, ...opts }); + await post(gh.api, REPO, run(), evil); + const refWrites = writes(gh.calls).filter((c) => c.path.startsWith("/git/refs")); + assert.ok(refWrites.length >= 1); + for (const c of refWrites) + assert.ok(c.path === `/git/refs/heads/${SHOTS_BRANCH}` ? c.body.force === false + : c.path === "/git/refs" && c.body.ref === "refs/heads/visual-diff-shots", JSON.stringify(c)); + if (!opts.branch) assert.deepEqual(gh.calls.find((c) => c.path === "/git/commits" && c.method === "POST").body.parents, []); + const trees = gh.calls.filter((c) => c.path === "/git/trees").flatMap((c) => c.body.tree.map((e) => e.path)); + assert.ok(trees.length && trees.every((p) => p.startsWith(`pr-7/${HEAD}/`)), trees.join()); + } + // Every write the script can make goes to the shots branch's objects or the pull request's comment. + const src = readFileSync(new URL("../scripts/visual-diff-post.mjs", import.meta.url), "utf8"); + assert.deepEqual([...src.matchAll(/api\("(POST|PATCH|PUT|DELETE)", ([^,)]+)/g)].map((m) => `${m[1]} ${m[2]}`).sort(), [ + "PATCH `/git/refs/heads/${SHOTS_BRANCH}`", "PATCH `/issues/comments/${id}`", "POST \"/git/blobs\"", "POST \"/git/commits\"", + "POST \"/git/refs\"", "POST \"/git/trees\"", "POST `/issues/${number}/comments`", + ]); + assert.match(src, /^export const SHOTS_BRANCH = "visual-diff-shots";$/m); + assert.match(src, /^const SHOTS_REF = `refs\/heads\/\$\{SHOTS_BRANCH\}`;$/m); +}); + +test("one marked comment: a later head rewrites the poster's own, not anyone else's", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + const pasted = { id: 1, user: { login: "someone" }, body: marker(OTHER) }; + const old = { id: 2, user: { login: "github-actions[bot]" }, body: `${marker(OTHER)}\nold shots` }; + const gh = fakeGitHub({ pulls: { 7: openPull() }, comments: [pasted, old] }); + assert.match(await post(gh.api, REPO, run(), dir), /updated the comment on #7/); + assert.equal(gh.state.comments[0].body, marker(OTHER)); + assert.ok(gh.state.comments[1].body.startsWith(`${marker(HEAD)}\n`)); + assert.equal(gh.state.comments.length, 2); + + const fresh = fakeGitHub({ pulls: { 7: openPull() }, comments: [pasted] }); + assert.match(await post(fresh.api, REPO, run(), dir), /posted the comment on #7/); + assert.equal(fresh.state.comments.filter((c) => c.body.includes("\n`)); + const raw = `https://raw.githubusercontent.com/${REPO}/${commit}/pr-7/${HEAD}`; + assert.ok(body.includes(`| ![base, 1440 x 900](${raw}/base/${WIDE}.png) | ![head, 1440 x 900](${raw}/head/${WIDE}.png) |`)); + assert.match(body, /#### 1440 x 900: changed/); + assert.match(body, /#### 375 x 812: the head didn't draw/); + assert.ok(body.includes(`| ![base, 375 x 812](${raw}/base/${NARROW}.png) | not drawn |`)); + assert.match(body, /1 of 2 shots changed\. The check failed/); + assert.match(body, /\[Visual diff run\]\(https:\/\/example\.test\/run\)/); + // An unknown conclusion adds nothing. + assert.doesNotMatch(commentBody({ repo: REPO, headSha: HEAD, commit, number: 7, runUrl: "u", conclusion: "x", shots: [] }), //); +}); + +test("the run comes from GitHub's event, and only a pull request's run counts", async () => { + const event = { workflow_run: { id: 42, event: "pull_request", head_sha: HEAD, html_url: "u", conclusion: "success", + pull_requests: [{ number: 8 }] } }; + assert.deepEqual(runFromEvent({ workflow_run: { ...event.workflow_run, head_branch: "change", head_repository: { full_name: "someone/fork" } } }), + { id: 42, headSha: HEAD, headRepo: "someone/fork", headBranch: "change", url: "u", conclusion: "success", pulls: [8] }); + assert.equal(runFromEvent({ workflow_run: { ...event.workflow_run, event: "push" } }), null); + assert.equal(runFromEvent({ workflow_run: { ...event.workflow_run, head_sha: "main" } }), null); + assert.equal(runFromEvent({}), null); + assert.equal(await find(fakeGitHub({ artifacts: [{ name: "visual-diff", expired: false }] }).api, run()), true); + assert.equal(await find(fakeGitHub({ artifacts: [] }).api, run()), false); + assert.equal(await find(fakeGitHub({ artifacts: [{ name: "visual-diff", expired: true }] }).api, run()), false); + assert.equal(await find(fakeGitHub().api, null), false); +}); + +test("the API client sends the token and the JSON body", async () => { + const seen = []; + const fetch = async (url, init) => { seen.push({ url, init }); return { status: 201, json: async () => ({ sha: "x" }) }; }; + const r = await github({ repo: REPO, token: "t0k", fetch })("POST", "/git/blobs", { content: "" }); + assert.deepEqual(r, { status: 201, data: { sha: "x" } }); + assert.equal(seen[0].url, `https://api.github.com/repos/${REPO}/git/blobs`); + assert.equal(seen[0].init.headers.Authorization, "Bearer t0k"); + assert.equal(seen[0].init.body, '{"content":""}'); +}); + +test("bad usage exits 2", () => { + const script = fileURLToPath(new URL("../scripts/visual-diff-post.mjs", import.meta.url)); + for (const args of [[], ["find", "x"], ["post"], ["push", "main"]]) + assert.equal(spawnSync(process.execPath, [script, ...args], { encoding: "utf8" }).status, 2, args.join(" ")); +}); + +// The workflow: after "Visual diff", from main, with the only write access in this repo's PR workflows. +const workflow = readFileSync(new URL("../.github/workflows/visual-diff-comment.yml", import.meta.url), "utf8"); +const code = workflow.replace(/#.*$/gm, ""); +const drawn = readFileSync(new URL("../.github/workflows/visual-diff.yml", import.meta.url), "utf8"); + +test("the poster runs after the Visual diff workflow completes, never on a pull request itself", () => { + const name = drawn.match(/^name: (.+)$/m)[1]; + assert.match(code, new RegExp(`^on:\\n {2}workflow_run:\\n {4}workflows: \\[${name}\\]\\n {4}types: \\[completed\\]\\n\\n`, "m")); + assert.doesNotMatch(code, /pull_request_target|^ {2}pull_request:|^ {2}push:/m); + assert.match(code, /^ {4}if: github\.event\.workflow_run\.event == 'pull_request'\n {4}runs-on: ubuntu-latest\n/m); + assert.equal([...code.matchAll(/runs-on:/g)].length, 1); +}); + +test("the poster checks out main only and runs nothing from the pull request", () => { + const checkouts = code.split(/^ {6}- /m).filter((s) => /actions\/checkout@/.test(s)); + assert.equal(checkouts.length, 1); + assert.match(checkouts[0], /\n {10}ref: main\n/); + assert.match(checkouts[0], /persist-credentials: false/); + assert.doesNotMatch(code, /git (push|fetch|checkout)|\bgh |ref: (?!main\n)/); + // The pull request's head names only the concurrency group. + for (const line of code.split("\n").filter((l) => /workflow_run\.head/.test(l))) assert.match(line, /^ {2}group: /, line); + // Event fields reach a shell only through the script, which reads GitHub's event file. + for (const block of code.matchAll(/^ {8}run: (.+)$/gm)) assert.doesNotMatch(block[1], /\$\{\{/, block[1]); + assert.match(code, /run: node scripts\/visual-diff-post\.mjs find >> "\$GITHUB_OUTPUT"/); + assert.match(code, /run: node scripts\/visual-diff-post\.mjs post "\$RUNNER_TEMP\/visual-diff" >> "\$GITHUB_STEP_SUMMARY"/); +}); + +test("only the poster job can write, and only what it needs", () => { + assert.match(code, /^permissions: \{\}\n/m); + assert.equal(code.match(/permissions:/g).length, 2); + assert.match(code, /^ {4}permissions:\n {6}actions: read *\n {6}contents: write *\n {6}pull-requests: write *\n {4}steps:/m); + assert.doesNotMatch(code, /secrets\.|id-token/); + const uses = [...code.matchAll(/uses:\s*(\S+)/g)].map((m) => m[1]); + assert.equal(uses.length, 3); + for (const u of uses) assert.match(u, /^actions\/[\w-]+@[0-9a-f]{40}$/, u); +}); From 427d99651b4915b4edf527c591fc574dde1691b7 Mon Sep 17 00:00:00 2001 From: Lyra Date: Mon, 2 Nov 2026 10:42:21 +0000 Subject: [PATCH 2/3] test: cover the poster's same-tree re-run, which makes no commit (#74) publish returns the shots branch's tip, with no commit and no ref write, when a re-run's tree equals the tip's. The fake GitHub now has a rerun mode that hands back the base tree, so this path is exercised; removing the guard fails the new test. Co-Authored-By: Claude Opus 5.5 --- test/visual-diff-post.test.mjs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/visual-diff-post.test.mjs b/test/visual-diff-post.test.mjs index 3837f9c..209ee8b 100644 --- a/test/visual-diff-post.test.mjs +++ b/test/visual-diff-post.test.mjs @@ -47,7 +47,7 @@ const good = (number = 7) => ({ }); // A fake GitHub: pulls by number, the shots branch, comments; records every request. -function fakeGitHub({ pulls = {}, branch = null, comments = [], moves = 0, artifacts = [] } = {}) { +function fakeGitHub({ pulls = {}, branch = null, comments = [], moves = 0, artifacts = [], rerun = false } = {}) { const calls = []; const state = { branch, comments: [...comments], n: 0, moves: 0 }; const sha = () => (++state.n).toString(16).padStart(40, "0"); @@ -62,7 +62,8 @@ function fakeGitHub({ pulls = {}, branch = null, comments = [], moves = 0, artif return state.branch ? reply(200, { object: { sha: state.branch } }) : reply(404, { message: "Not Found" }); if (method === "GET" && (m = path.match(/^\/git\/commits\/(\w+)$/))) return reply(200, { tree: { sha: `tree-of-${m[1]}` } }); if (method === "POST" && path === "/git/blobs") return reply(201, { sha: sha() }); - if (method === "POST" && path === "/git/trees") return reply(201, { sha: sha() }); + // rerun: the shots are already on the branch, so GitHub hands back the tip's own tree. + if (method === "POST" && path === "/git/trees") return reply(201, { sha: rerun && body.base_tree ? body.base_tree : sha() }); if (method === "POST" && path === "/git/commits") return reply(201, { sha: sha() }); if (method === "POST" && path === "/git/refs") { if (state.branch) return reply(422, { message: "Reference already exists" }); @@ -185,6 +186,17 @@ test("the only ref it writes is refs/heads/visual-diff-shots, whatever the artif assert.match(src, /^const SHOTS_REF = `refs\/heads\/\$\{SHOTS_BRANCH\}`;$/m); }); +test("a re-run with the same shots makes no commit and links the tip that holds them", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + const tip = "c".repeat(40); + const gh = fakeGitHub({ pulls: { 7: openPull() }, branch: tip, rerun: true }); + assert.match(await post(gh.api, REPO, run(), dir), new RegExp(`on ${SHOTS_BRANCH} at \`${tip}\`; posted the comment on #7`)); + assert.deepEqual(gh.calls.filter((c) => c.path === "/git/commits" || c.path.startsWith("/git/refs")).filter((c) => c.method !== "GET"), []); + assert.equal(gh.state.branch, tip); + assert.ok(gh.state.comments[0].body.includes(`https://raw.githubusercontent.com/${REPO}/${tip}/pr-7/${HEAD}/`)); +}); + test("one marked comment: a later head rewrites the poster's own, not anyone else's", async (t) => { const dir = await artifact(good(7)); t.after(() => rm(dir, { recursive: true, force: true })); From cc054c22973895de857eea3eb24da5d68f4ce27c Mon Sep 17 00:00:00 2001 From: Lyra Date: Mon, 2 Nov 2026 10:42:21 +0000 Subject: [PATCH 3/3] docs: a PR that changes SHOTS gets no visual-diff comment until it merges (#74) The poster runs main's code, so its artifact allowlist is main's SHOTS. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 27354f5..87baa45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,8 @@ a laptop. one comment marked ``. The artifact is untrusted (a fork's pull request can change the workflow that made it): never check out or run the pull request's code there, never put text from the artifact in the comment, and never write any ref but `visual-diff-shots`. + The poster reads only the files main's `SHOTS` (in `scripts/visual-diff.mjs`) names, so a pull request + that changes `SHOTS` gets no comment ("unexpected file in the artifact") until it merges. - `test/`: `node --test` suites for the pure modules and the page (`day.test.mjs` for Day). - `test/worker/`: vitest suites run in workerd by `@cloudflare/vitest-plugin` (`vitest.config.mjs` reads `wrangler.jsonc`). They fake Access's certs with a throwaway key and drive every route