diff --git a/.github/workflows/visual-diff-comment.yml b/.github/workflows/visual-diff-comment.yml new file mode 100644 index 0000000..5980390 --- /dev/null +++ b/.github/workflows/visual-diff-comment.yml @@ -0,0 +1,55 @@ +name: Visual diff comment + +# setkernel-live#74, slice 2: posts the shots that "Visual diff" (visual-diff.yml) kept, in one +# comment on the pull request marked . That run has a read-only token, +# a fork's included, so it can't post; this one runs from main after it completes, with this repo's +# token. It never checks out or runs the pull request's code: scripts/visual-diff-post.mjs (main's) +# treats the artifact as untrusted, uses the pull request only if its head is the run's head, and +# writes only to the visual-diff-shots branch. Not a required check: visual-diff is. +on: + workflow_run: + workflows: [Visual diff] + types: [completed] + +permissions: {} + +# A newer run for the same branch supersedes this one; pushes from different pull requests to +# visual-diff-shots retry on their own (publish in the script). +concurrency: + group: visual-diff-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + +jobs: + # Public repo: what a fork's pull request made is handled here, so never a self-hosted runner. + post: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read # the run's artifact + contents: write # visual-diff-shots, and only that ref (scripts/visual-diff-post.mjs) + pull-requests: write # the comment + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + # No artifact when no UI path changed: nothing to post. + - id: find + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/visual-diff-post.mjs find >> "$GITHUB_OUTPUT" + - if: steps.find.outputs.artifact == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: visual-diff + path: ${{ runner.temp }}/visual-diff + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - if: steps.find.outputs.artifact == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/visual-diff-post.mjs post "$RUNNER_TEMP/visual-diff" >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 7ff24f7..87baa45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ Public repo: everything here is visible to anyone. # headline/demo/replay, Day's lanes/Now column/key/scrubbing/fitWall, # Buzz alert, page hosts, fonts and contrast, the smoke check, the showcase's # publisher, the workflow runner guard, the visual-diff check's plan and workflow, +# its poster (the artifact, the pull request, the shots branch, the comment), # and repo names against GitHub's public list, live) # worker: npm run test:worker (vitest in workerd: the real Worker and Hub, see test/worker/) # lint: npm run check (node --check on every script) @@ -69,6 +70,14 @@ a laptop. from the head at 1440x900 and 375x812, on one runner with a pinned Playwright, and keeps the shots as the run's `visual-diff` artifact. It fails only when the head page errs. It always runs, and passes with no UI change, so it can be a required check: never give it a `paths:` filter or a job `if:`. +- `scripts/visual-diff-post.mjs` and `.github/workflows/visual-diff-comment.yml`: its poster. A + `workflow_run` after "Visual diff", from main, with the only write token a pull request's run can + reach: it commits the shots to the orphan branch `visual-diff-shots` (`pr-//`) and keeps + one comment marked ``. The artifact is untrusted (a fork's pull request + can change the workflow that made it): never check out or run the pull request's code there, never + put text from the artifact in the comment, and never write any ref but `visual-diff-shots`. + The poster reads only the files main's `SHOTS` (in `scripts/visual-diff.mjs`) names, so a pull request + that changes `SHOTS` gets no comment ("unexpected file in the artifact") until it merges. - `test/`: `node --test` suites for the pure modules and the page (`day.test.mjs` for Day). - `test/worker/`: vitest suites run in workerd by `@cloudflare/vitest-plugin` (`vitest.config.mjs` reads `wrangler.jsonc`). They fake Access's certs with a throwaway key and drive every route diff --git a/scripts/visual-diff-post.mjs b/scripts/visual-diff-post.mjs new file mode 100644 index 0000000..dbe403d --- /dev/null +++ b/scripts/visual-diff-post.mjs @@ -0,0 +1,239 @@ +// The visual-diff check's poster (setkernel-live#74, slice 2). .github/workflows/visual-diff.yml runs +// on a pull request with a read-only token, a fork's included, so it can only keep its shots as the +// run's visual-diff artifact. .github/workflows/visual-diff-comment.yml runs this, from main, when +// that run completes: it commits the shots to the visual-diff-shots branch, under +// pr-//, and posts them in one comment on the pull request, marked +// . A later head rewrites the same comment. +// +// node scripts/visual-diff-post.mjs find prints artifact=true or artifact=false +// node scripts/visual-diff-post.mjs post publishes , the downloaded artifact; prints a summary +// +// Both read the workflow_run event from GITHUB_EVENT_PATH, and GITHUB_REPOSITORY and GITHUB_TOKEN. +// +// The artifact is untrusted: a pull request, a fork's too, can change the workflow that made it. So +// only the expected PNGs are read, each checked for its signature, size and dimensions; nothing in +// it is run, and no text from it reaches the comment. Its pull request number is only a candidate: +// the pull request is used only if it is open here and its head is the run's head (the event's +// head_sha). The only ref this ever writes is refs/heads/visual-diff-shots. + +import { lstat, readFile, readdir } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { SHOTS } from "./visual-diff.mjs"; + +export const SHOTS_BRANCH = "visual-diff-shots"; +const SHOTS_REF = `refs/heads/${SHOTS_BRANCH}`; +export const MAX_PNG = 5 * 1024 * 1024; +const MAX_JSON = 64 * 1024; +const SHA = /^[0-9a-f]{40}$/; +const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const BOT = "github-actions[bot]"; + +export const marker = (sha) => ``; +const MARKED = //; + +// A PNG of exactly width x height: its signature, then IHDR, which holds the size. +export function checkPng(buf, width, height) { + if (buf.length > MAX_PNG) return `over ${MAX_PNG} bytes`; + if (buf.length < 24 || !buf.subarray(0, 8).equals(SIGNATURE) || buf.toString("latin1", 12, 16) !== "IHDR") + return "not a PNG"; + const [w, h] = [buf.readUInt32BE(16), buf.readUInt32BE(20)]; + return w === width && h === height ? null : `${w} x ${h}, not ${width} x ${height}`; +} + +// The downloaded artifact: base/.png, head/.png and visual-diff.json, nothing else, no +// links. A shot that failed to draw has no PNG. Throws on anything unexpected. +export async function readArtifact(dir) { + const allowed = new Set(["visual-diff.json", "base", "head", + ...SHOTS.flatMap(([name]) => [`base/${name}.png`, `head/${name}.png`])]); + const walk = async (rel) => { + for (const name of await readdir(join(dir, rel))) { + const path = rel ? `${rel}/${name}` : name; + if (!allowed.has(path)) throw new Error(`unexpected file in the artifact: ${JSON.stringify(path)}`); + const st = await lstat(join(dir, path)); + const folder = path === "base" || path === "head"; + if (folder ? !st.isDirectory() : !st.isFile()) throw new Error(`not a plain ${folder ? "folder" : "file"}: ${path}`); + if (folder) await walk(path); + } + }; + await walk(""); + const read = async (path, max) => { + const st = await lstat(join(dir, path)).catch(() => null); + if (!st) return null; + if (st.size > max) throw new Error(`${path} is over ${max} bytes`); + return readFile(join(dir, path)); + }; + const shots = []; + for (const [name, width, height] of SHOTS) { + const shot = { name, width, height }; + for (const side of ["base", "head"]) { + const png = await read(`${side}/${name}.png`, MAX_PNG); + const bad = png && checkPng(png, width, height); + if (bad) throw new Error(`${side}/${name}.png: ${bad}`); + shot[side] = png; + } + shot.changed = !!(shot.base && shot.head) && !shot.base.equals(shot.head); + shots.push(shot); + } + const json = await read("visual-diff.json", MAX_JSON); + let number = null; + try { + const n = JSON.parse(json)?.pr?.number; + if (Number.isSafeInteger(n) && n > 0) number = n; + } catch {} + return { number, shots }; +} + +// The REST API with the job's token. fetch is a parameter so the tests can stand in for GitHub. +export function github({ repo, token, fetch = globalThis.fetch, base = "https://api.github.com" }) { + return async (method, path, body) => { + const res = await fetch(`${base}/repos/${repo}${path}`, { + method, + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + ...(body ? { "Content-Type": "application/json" } : {}) }, + body: body ? JSON.stringify(body) : undefined, + }); + const data = res.status === 204 ? null : await res.json().catch(() => null); + return { status: res.status, data }; + }; +} + +const ok = (r, what) => { + if (r.status >= 200 && r.status < 300) return r.data; + throw new Error(`${what}: HTTP ${r.status}${r.data?.message ? ` (${r.data.message})` : ""}`); +}; + +// The open pull request here whose head is the run's: the same commit, from the same repo and +// branch, so another pull request on the same commit can't take its comment. Candidates come from +// the event and the artifact; each is checked against GitHub. null if none matches (closed, moved +// on, or a lie). +export async function findPull(api, repo, run) { + for (const n of new Set([...run.pulls, ...run.claimed].filter((n) => Number.isSafeInteger(n) && n > 0))) { + const r = await api("GET", `/pulls/${n}`); + if (r.status === 404) continue; + const pr = ok(r, `pull request ${n}`); + if (pr.state === "open" && pr.base?.repo?.full_name === repo && pr.head?.sha === run.headSha && + pr.head?.repo?.full_name === run.headRepo && pr.head?.ref === run.headBranch) return n; + } + return null; +} + +// Commits the shots under pr-// on top of visual-diff-shots (an orphan branch, made on the +// first run) and returns the commit that holds them. Another pull request's poster may move the +// branch meanwhile, so a rejected update starts again from the new tip. +export async function publish(api, number, headSha, shots, tries = 5) { + const files = shots.flatMap((s) => ["base", "head"].filter((side) => s[side]) + .map((side) => ({ path: `pr-${number}/${headSha}/${side}/${s.name}.png`, png: s[side] }))); + const tree = []; + for (const f of files) { + const blob = ok(await api("POST", "/git/blobs", { content: f.png.toString("base64"), encoding: "base64" }), "blob"); + tree.push({ path: f.path, mode: "100644", type: "blob", sha: blob.sha }); + } + for (let i = 0; i < tries; i++) { + const ref = await api("GET", `/git/ref/heads/${SHOTS_BRANCH}`); + const parent = ref.status === 404 ? null : ok(ref, SHOTS_BRANCH).object.sha; + const parentTree = parent && ok(await api("GET", `/git/commits/${parent}`), "commit").tree.sha; + const made = ok(await api("POST", "/git/trees", { ...(parentTree ? { base_tree: parentTree } : {}), tree }), "tree"); + if (made.sha === parentTree) return parent; // a re-run: these shots are already there + const commit = ok(await api("POST", "/git/commits", { + message: `visual-diff: pr-${number} ${headSha}`, tree: made.sha, parents: parent ? [parent] : [], + }), "commit").sha; + const moved = parent + ? await api("PATCH", `/git/refs/heads/${SHOTS_BRANCH}`, { sha: commit, force: false }) + : await api("POST", "/git/refs", { ref: SHOTS_REF, sha: commit }); + if (moved.status === 422 || moved.status === 409) continue; // not a fast-forward, or made meanwhile + ok(moved, `update ${SHOTS_BRANCH}`); + return commit; + } + throw new Error(`${SHOTS_BRANCH} kept moving; gave up after ${tries} tries`); +} + +const CONCLUSION = { + success: "The check passed.", + failure: "The check failed: the head didn't draw cleanly. The run's summary says why.", +}; + +// The comment, from this file and the event only: no text from the artifact. +export function commentBody({ repo, headSha, commit, number, runUrl, conclusion, shots }) { + const raw = (side, name) => `https://raw.githubusercontent.com/${repo}/${commit}/pr-${number}/${headSha}/${side}/${name}.png`; + const cell = (s, side) => s[side] ? `![${side}, ${s.width} x ${s.height}](${raw(side, s.name)})` : "not drawn"; + const state = (s) => !s.head ? "the head didn't draw" : !s.base ? "the base didn't draw" : s.changed ? "changed" : "same"; + const changed = shots.filter((s) => s.changed).length; + return [ + marker(headSha), + "### Visual diff", + "", + `\`?demo\` drawn from this pull request's base and from its head, \`${headSha.slice(0, 7)}\`, in the ` + + `[Visual diff run](${runUrl}). ${changed ? `${changed} of ${shots.length} shots changed.` : "No shot changed."}` + + (CONCLUSION[conclusion] ? ` ${CONCLUSION[conclusion]}` : ""), + ...shots.flatMap((s) => [ + "", + `#### ${s.width} x ${s.height}: ${state(s)}`, + "", + "| Base | Head |", + "| --- | --- |", + `| ${cell(s, "base")} | ${cell(s, "head")} |`, + ]), + "", + ].join("\n"); +} + +// The poster's own marked comment, whatever head it names. Only the bot's: anyone can paste a marker. +export async function findComment(api, number) { + for (let page = 1; page <= 30; page++) { + const list = ok(await api("GET", `/issues/${number}/comments?per_page=100&page=${page}`), "comments"); + const hit = list.find((c) => c.user?.login === BOT && MARKED.test(c.body ?? "")); + if (hit) return hit.id; + if (list.length < 100) return null; + } + return null; +} + +export async function comment(api, number, body) { + const id = await findComment(api, number); + if (id) ok(await api("PATCH", `/issues/comments/${id}`, { body }), "update comment"); + else ok(await api("POST", `/issues/${number}/comments`, { body }), "post comment"); + return id ? "updated" : "posted"; +} + +// The completed run, from the event GitHub wrote (not from the artifact). +export function runFromEvent(event) { + const run = event?.workflow_run; + if (!run || run.event !== "pull_request" || !SHA.test(run.head_sha ?? "")) return null; + return { id: run.id, headSha: run.head_sha, headRepo: run.head_repository?.full_name, headBranch: run.head_branch, + url: run.html_url, conclusion: run.conclusion, pulls: (run.pull_requests ?? []).map((p) => p.number) }; +} + +// find: did that run keep a visual-diff artifact? (None when no UI path changed.) +export async function find(api, run) { + if (!run) return false; + const r = ok(await api("GET", `/actions/runs/${run.id}/artifacts?name=visual-diff`), "artifacts"); + return (r.artifacts ?? []).some((a) => a.name === "visual-diff" && !a.expired); +} + +export async function post(api, repo, run, dir) { + if (!run) return "Not a pull request's run, so nothing was posted."; + const { number: claimed, shots } = await readArtifact(dir); + const number = await findPull(api, repo, { ...run, claimed: [claimed] }); + if (!number) return `No open pull request has head \`${run.headSha}\` any more, so nothing was posted.`; + if (!shots.some((s) => s.base || s.head)) return "The artifact holds no shots, so nothing was posted."; + const commit = await publish(api, number, run.headSha, shots); + const body = commentBody({ repo, headSha: run.headSha, commit, number, runUrl: run.url, conclusion: run.conclusion, shots }); + return `Shots for \`${run.headSha}\` are on ${SHOTS_BRANCH} at \`${commit}\`; ${await comment(api, number, body)} the comment on #${number}.`; +} + +// Not import.meta.main: that needs Node 24.2 (scripts/smoke.mjs says the same). +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [cmd, ...args] = process.argv.slice(2); + if ((cmd === "find" && !args.length) || (cmd === "post" && args.length === 1)) { + const repo = process.env.GITHUB_REPOSITORY; + const run = runFromEvent(JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"))); + const api = github({ repo, token: process.env.GITHUB_TOKEN }); + if (cmd === "find") console.log(`artifact=${await find(api, run)}`); + else console.log(`### Visual diff comment\n\n${await post(api, repo, run, args[0])}`); + } else { + console.error("usage: visual-diff-post.mjs find | visual-diff-post.mjs post "); + process.exitCode = 2; + } +} diff --git a/test/visual-diff-post.test.mjs b/test/visual-diff-post.test.mjs new file mode 100644 index 0000000..209ee8b --- /dev/null +++ b/test/visual-diff-post.test.mjs @@ -0,0 +1,298 @@ +// The visual-diff check's poster (setkernel-live#74, slice 2): it reads only the expected PNGs from +// the untrusted artifact, posts only on the open pull request whose head is the run's head, writes +// only refs/heads/visual-diff-shots, and keeps one marked comment; and its workflow's shape. GitHub +// is a fake here. Run: npm test +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { SHOTS } from "../scripts/visual-diff.mjs"; +import { + MAX_PNG, SHOTS_BRANCH, checkPng, commentBody, find, github, marker, post, readArtifact, runFromEvent, +} from "../scripts/visual-diff-post.mjs"; + +const REPO = "setkernel/setkernel-live"; +const HEAD = "a".repeat(40); +const OTHER = "b".repeat(40); +const [[WIDE, W, H], [NARROW, w, h]] = SHOTS; + +const png = (width, height, fill = 0) => { + const b = Buffer.alloc(64, fill); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(b); + b.writeUInt32BE(13, 8); + b.write("IHDR", 12, "latin1"); + b.writeUInt32BE(width, 16); + b.writeUInt32BE(height, 20); + return b; +}; + +// An artifact as visual-diff.yml uploads it: { "base/.png": Buffer, ..., "visual-diff.json": "..." }. +async function artifact(files) { + const dir = await mkdtemp(join(process.env.TMPDIR ?? tmpdir(), "vd-")); + for (const [path, body] of Object.entries(files)) { + if (body === undefined) continue; + await mkdir(join(dir, path, ".."), { recursive: true }); + await writeFile(join(dir, path), body); + } + return dir; +} +const good = (number = 7) => ({ + [`base/${WIDE}.png`]: png(W, H), [`head/${WIDE}.png`]: png(W, H, 1), + [`base/${NARROW}.png`]: png(w, h), [`head/${NARROW}.png`]: png(w, h), + "visual-diff.json": JSON.stringify({ pr: { number, head: HEAD }, shots: [] }), +}); + +// A fake GitHub: pulls by number, the shots branch, comments; records every request. +function fakeGitHub({ pulls = {}, branch = null, comments = [], moves = 0, artifacts = [], rerun = false } = {}) { + const calls = []; + const state = { branch, comments: [...comments], n: 0, moves: 0 }; + const sha = () => (++state.n).toString(16).padStart(40, "0"); + const api = async (method, path, body) => { + calls.push({ method, path, body }); + const reply = (status, data) => ({ status, data }); + let m; + if (method === "GET" && (m = path.match(/^\/pulls\/(\d+)$/))) + return pulls[m[1]] ? reply(200, pulls[m[1]]) : reply(404, { message: "Not Found" }); + if (method === "GET" && path.startsWith("/actions/runs/")) return reply(200, { artifacts }); + if (method === "GET" && path === `/git/ref/heads/${SHOTS_BRANCH}`) + return state.branch ? reply(200, { object: { sha: state.branch } }) : reply(404, { message: "Not Found" }); + if (method === "GET" && (m = path.match(/^\/git\/commits\/(\w+)$/))) return reply(200, { tree: { sha: `tree-of-${m[1]}` } }); + if (method === "POST" && path === "/git/blobs") return reply(201, { sha: sha() }); + // rerun: the shots are already on the branch, so GitHub hands back the tip's own tree. + if (method === "POST" && path === "/git/trees") return reply(201, { sha: rerun && body.base_tree ? body.base_tree : sha() }); + if (method === "POST" && path === "/git/commits") return reply(201, { sha: sha() }); + if (method === "POST" && path === "/git/refs") { + if (state.branch) return reply(422, { message: "Reference already exists" }); + state.branch = body.sha; + return reply(201, {}); + } + if (method === "PATCH" && path === `/git/refs/heads/${SHOTS_BRANCH}`) { + if (state.moves < moves) { state.moves++; state.branch = sha(); return reply(422, { message: "not a fast forward" }); } + state.branch = body.sha; + return reply(200, {}); + } + if (method === "GET" && (m = path.match(/^\/issues\/(\d+)\/comments\?/))) return reply(200, state.comments); + if (method === "POST" && (m = path.match(/^\/issues\/(\d+)\/comments$/))) { + state.comments.push({ id: 99, user: { login: "github-actions[bot]" }, body: body.body }); + return reply(201, {}); + } + if (method === "PATCH" && (m = path.match(/^\/issues\/comments\/(\d+)$/))) { + const c = state.comments.find((c) => c.id === Number(m[1])); + c.body = body.body; + return reply(200, {}); + } + return reply(500, { message: `unexpected ${method} ${path}` }); + }; + return { api, calls, state }; +} +const openPull = (sha = HEAD, head = {}) => + ({ state: "open", head: { sha, ref: "change", repo: { full_name: "someone/fork" }, ...head }, base: { repo: { full_name: REPO } } }); +const run = (over = {}) => ({ id: 42, headSha: HEAD, headRepo: "someone/fork", headBranch: "change", + url: "https://github.com/setkernel/setkernel-live/actions/runs/42", conclusion: "success", pulls: [], ...over }); +const writes = (calls) => calls.filter((c) => c.method !== "GET"); + +test("a PNG must have the shot's exact size and stay under the cap", () => { + assert.equal(checkPng(png(W, H), W, H), null); + assert.equal(checkPng(png(w, h), W, H), `${w} x ${h}, not ${W} x ${H}`); + assert.equal(checkPng(Buffer.from("".padEnd(64)), W, H), "not a PNG"); + assert.equal(checkPng(Buffer.alloc(10), W, H), "not a PNG"); + assert.equal(checkPng(Buffer.concat([png(W, H), Buffer.alloc(MAX_PNG)]), W, H), `over ${MAX_PNG} bytes`); +}); + +test("the artifact is read as the expected PNGs and nothing else", async (t) => { + const dirs = []; + t.after(() => Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true })))); + const make = async (files) => { const d = await artifact(files); dirs.push(d); return d; }; + + const { number, shots } = await readArtifact(await make(good())); + assert.equal(number, 7); + assert.deepEqual(shots.map((s) => [s.name, s.changed]), [[WIDE, true], [NARROW, false]]); + + // A shot that didn't draw has no PNG; a JSON that isn't one only loses the candidate number. + const { [`head/${NARROW}.png`]: _, ...missing } = good(); + const partial = await readArtifact(await make({ ...missing, "visual-diff.json": "{not json" })); + assert.equal(partial.number, null); + assert.equal(partial.shots[1].head, null); + assert.equal(partial.shots[1].changed, false); + for (const n of [0, -1, 1.5, "7", 2 ** 60]) { + const r = await readArtifact(await make({ ...good(), "visual-diff.json": JSON.stringify({ pr: { number: n } }) })); + assert.equal(r.number, null, String(n)); + } + + for (const [files, why] of [ + [{ ...good(), "base/run.sh": "echo" }, /unexpected file.*base\/run\.sh/], + [{ ...good(), "head/x/y.png": png(W, H) }, /unexpected file.*head\/x/], + [{ ...missing, "visual-diff.json": undefined, "visual-diff.json/a": "" }, /not a plain file: visual-diff\.json/], + [{ ...good(), [`head/${WIDE}.png`]: png(w, h) }, /head\/demo-1440x900\.png: 375 x 812, not 1440 x 900/], + [{ ...good(), [`base/${NARROW}.png`]: "not a png" }, /not a PNG/], + [{ ...good(), "visual-diff.json": "x".repeat(70_000) }, /over/], + ]) await assert.rejects(readArtifact(await make(files)), why); + + const linked = await make(good()); + await rm(join(linked, `head/${WIDE}.png`)); + await symlink("/etc/passwd", join(linked, `head/${WIDE}.png`)); + await assert.rejects(readArtifact(linked), /not a plain file/); +}); + +test("it posts only on the open pull request whose head is the run's head", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + // Gone, moved on, closed, into another repo, or the same commit from another fork or branch. + for (const pulls of [{}, { 7: openPull(OTHER) }, { 7: { ...openPull(), state: "closed" } }, + { 7: { ...openPull(), base: { repo: { full_name: "someone/fork" } } } }, + { 7: openPull(HEAD, { repo: { full_name: "victim/fork" } }) }, { 7: openPull(HEAD, { ref: "other" }) }, + { 7: openPull(HEAD, { repo: null }) }]) { + const gh = fakeGitHub({ pulls }); + assert.match(await post(gh.api, REPO, run(), dir), /nothing was posted/); + assert.deepEqual(writes(gh.calls), [], "no write when the pull request doesn't match"); + } + // The event's pull request is tried first, and a false claim in the artifact is skipped. + const gh = fakeGitHub({ pulls: { 7: openPull(OTHER), 8: openPull() } }); + assert.match(await post(gh.api, REPO, run({ pulls: [8] }), dir), /posted the comment on #8/); + assert.ok(writes(gh.calls).every((c) => !c.path.includes("/issues/7/"))); + // A fork's run has no pull request in the event: the artifact's candidate, once checked, is used. + const fork = fakeGitHub({ pulls: { 5: { ...openPull(), state: "closed" }, 7: openPull() } }); + assert.match(await post(fork.api, REPO, run({ pulls: [5] }), dir), /posted the comment on #7/); + assert.equal(await post(fakeGitHub().api, REPO, null, dir), "Not a pull request's run, so nothing was posted."); +}); + +test("the only ref it writes is refs/heads/visual-diff-shots, whatever the artifact says", async (t) => { + const evil = await artifact({ ...good(7), "visual-diff.json": JSON.stringify({ pr: { number: 7, head: "main", ref: "refs/heads/main" } }) }); + t.after(() => rm(evil, { recursive: true, force: true })); + // First run: the branch is made as an orphan. Later: a fast-forward, retried when another poster moved it. + for (const opts of [{}, { branch: "c".repeat(40) }, { branch: "c".repeat(40), moves: 2 }]) { + const gh = fakeGitHub({ pulls: { 7: openPull() }, ...opts }); + await post(gh.api, REPO, run(), evil); + const refWrites = writes(gh.calls).filter((c) => c.path.startsWith("/git/refs")); + assert.ok(refWrites.length >= 1); + for (const c of refWrites) + assert.ok(c.path === `/git/refs/heads/${SHOTS_BRANCH}` ? c.body.force === false + : c.path === "/git/refs" && c.body.ref === "refs/heads/visual-diff-shots", JSON.stringify(c)); + if (!opts.branch) assert.deepEqual(gh.calls.find((c) => c.path === "/git/commits" && c.method === "POST").body.parents, []); + const trees = gh.calls.filter((c) => c.path === "/git/trees").flatMap((c) => c.body.tree.map((e) => e.path)); + assert.ok(trees.length && trees.every((p) => p.startsWith(`pr-7/${HEAD}/`)), trees.join()); + } + // Every write the script can make goes to the shots branch's objects or the pull request's comment. + const src = readFileSync(new URL("../scripts/visual-diff-post.mjs", import.meta.url), "utf8"); + assert.deepEqual([...src.matchAll(/api\("(POST|PATCH|PUT|DELETE)", ([^,)]+)/g)].map((m) => `${m[1]} ${m[2]}`).sort(), [ + "PATCH `/git/refs/heads/${SHOTS_BRANCH}`", "PATCH `/issues/comments/${id}`", "POST \"/git/blobs\"", "POST \"/git/commits\"", + "POST \"/git/refs\"", "POST \"/git/trees\"", "POST `/issues/${number}/comments`", + ]); + assert.match(src, /^export const SHOTS_BRANCH = "visual-diff-shots";$/m); + assert.match(src, /^const SHOTS_REF = `refs\/heads\/\$\{SHOTS_BRANCH\}`;$/m); +}); + +test("a re-run with the same shots makes no commit and links the tip that holds them", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + const tip = "c".repeat(40); + const gh = fakeGitHub({ pulls: { 7: openPull() }, branch: tip, rerun: true }); + assert.match(await post(gh.api, REPO, run(), dir), new RegExp(`on ${SHOTS_BRANCH} at \`${tip}\`; posted the comment on #7`)); + assert.deepEqual(gh.calls.filter((c) => c.path === "/git/commits" || c.path.startsWith("/git/refs")).filter((c) => c.method !== "GET"), []); + assert.equal(gh.state.branch, tip); + assert.ok(gh.state.comments[0].body.includes(`https://raw.githubusercontent.com/${REPO}/${tip}/pr-7/${HEAD}/`)); +}); + +test("one marked comment: a later head rewrites the poster's own, not anyone else's", async (t) => { + const dir = await artifact(good(7)); + t.after(() => rm(dir, { recursive: true, force: true })); + const pasted = { id: 1, user: { login: "someone" }, body: marker(OTHER) }; + const old = { id: 2, user: { login: "github-actions[bot]" }, body: `${marker(OTHER)}\nold shots` }; + const gh = fakeGitHub({ pulls: { 7: openPull() }, comments: [pasted, old] }); + assert.match(await post(gh.api, REPO, run(), dir), /updated the comment on #7/); + assert.equal(gh.state.comments[0].body, marker(OTHER)); + assert.ok(gh.state.comments[1].body.startsWith(`${marker(HEAD)}\n`)); + assert.equal(gh.state.comments.length, 2); + + const fresh = fakeGitHub({ pulls: { 7: openPull() }, comments: [pasted] }); + assert.match(await post(fresh.api, REPO, run(), dir), /posted the comment on #7/); + assert.equal(fresh.state.comments.filter((c) => c.body.includes("\n`)); + const raw = `https://raw.githubusercontent.com/${REPO}/${commit}/pr-7/${HEAD}`; + assert.ok(body.includes(`| ![base, 1440 x 900](${raw}/base/${WIDE}.png) | ![head, 1440 x 900](${raw}/head/${WIDE}.png) |`)); + assert.match(body, /#### 1440 x 900: changed/); + assert.match(body, /#### 375 x 812: the head didn't draw/); + assert.ok(body.includes(`| ![base, 375 x 812](${raw}/base/${NARROW}.png) | not drawn |`)); + assert.match(body, /1 of 2 shots changed\. The check failed/); + assert.match(body, /\[Visual diff run\]\(https:\/\/example\.test\/run\)/); + // An unknown conclusion adds nothing. + assert.doesNotMatch(commentBody({ repo: REPO, headSha: HEAD, commit, number: 7, runUrl: "u", conclusion: "x", shots: [] }), //); +}); + +test("the run comes from GitHub's event, and only a pull request's run counts", async () => { + const event = { workflow_run: { id: 42, event: "pull_request", head_sha: HEAD, html_url: "u", conclusion: "success", + pull_requests: [{ number: 8 }] } }; + assert.deepEqual(runFromEvent({ workflow_run: { ...event.workflow_run, head_branch: "change", head_repository: { full_name: "someone/fork" } } }), + { id: 42, headSha: HEAD, headRepo: "someone/fork", headBranch: "change", url: "u", conclusion: "success", pulls: [8] }); + assert.equal(runFromEvent({ workflow_run: { ...event.workflow_run, event: "push" } }), null); + assert.equal(runFromEvent({ workflow_run: { ...event.workflow_run, head_sha: "main" } }), null); + assert.equal(runFromEvent({}), null); + assert.equal(await find(fakeGitHub({ artifacts: [{ name: "visual-diff", expired: false }] }).api, run()), true); + assert.equal(await find(fakeGitHub({ artifacts: [] }).api, run()), false); + assert.equal(await find(fakeGitHub({ artifacts: [{ name: "visual-diff", expired: true }] }).api, run()), false); + assert.equal(await find(fakeGitHub().api, null), false); +}); + +test("the API client sends the token and the JSON body", async () => { + const seen = []; + const fetch = async (url, init) => { seen.push({ url, init }); return { status: 201, json: async () => ({ sha: "x" }) }; }; + const r = await github({ repo: REPO, token: "t0k", fetch })("POST", "/git/blobs", { content: "" }); + assert.deepEqual(r, { status: 201, data: { sha: "x" } }); + assert.equal(seen[0].url, `https://api.github.com/repos/${REPO}/git/blobs`); + assert.equal(seen[0].init.headers.Authorization, "Bearer t0k"); + assert.equal(seen[0].init.body, '{"content":""}'); +}); + +test("bad usage exits 2", () => { + const script = fileURLToPath(new URL("../scripts/visual-diff-post.mjs", import.meta.url)); + for (const args of [[], ["find", "x"], ["post"], ["push", "main"]]) + assert.equal(spawnSync(process.execPath, [script, ...args], { encoding: "utf8" }).status, 2, args.join(" ")); +}); + +// The workflow: after "Visual diff", from main, with the only write access in this repo's PR workflows. +const workflow = readFileSync(new URL("../.github/workflows/visual-diff-comment.yml", import.meta.url), "utf8"); +const code = workflow.replace(/#.*$/gm, ""); +const drawn = readFileSync(new URL("../.github/workflows/visual-diff.yml", import.meta.url), "utf8"); + +test("the poster runs after the Visual diff workflow completes, never on a pull request itself", () => { + const name = drawn.match(/^name: (.+)$/m)[1]; + assert.match(code, new RegExp(`^on:\\n {2}workflow_run:\\n {4}workflows: \\[${name}\\]\\n {4}types: \\[completed\\]\\n\\n`, "m")); + assert.doesNotMatch(code, /pull_request_target|^ {2}pull_request:|^ {2}push:/m); + assert.match(code, /^ {4}if: github\.event\.workflow_run\.event == 'pull_request'\n {4}runs-on: ubuntu-latest\n/m); + assert.equal([...code.matchAll(/runs-on:/g)].length, 1); +}); + +test("the poster checks out main only and runs nothing from the pull request", () => { + const checkouts = code.split(/^ {6}- /m).filter((s) => /actions\/checkout@/.test(s)); + assert.equal(checkouts.length, 1); + assert.match(checkouts[0], /\n {10}ref: main\n/); + assert.match(checkouts[0], /persist-credentials: false/); + assert.doesNotMatch(code, /git (push|fetch|checkout)|\bgh |ref: (?!main\n)/); + // The pull request's head names only the concurrency group. + for (const line of code.split("\n").filter((l) => /workflow_run\.head/.test(l))) assert.match(line, /^ {2}group: /, line); + // Event fields reach a shell only through the script, which reads GitHub's event file. + for (const block of code.matchAll(/^ {8}run: (.+)$/gm)) assert.doesNotMatch(block[1], /\$\{\{/, block[1]); + assert.match(code, /run: node scripts\/visual-diff-post\.mjs find >> "\$GITHUB_OUTPUT"/); + assert.match(code, /run: node scripts\/visual-diff-post\.mjs post "\$RUNNER_TEMP\/visual-diff" >> "\$GITHUB_STEP_SUMMARY"/); +}); + +test("only the poster job can write, and only what it needs", () => { + assert.match(code, /^permissions: \{\}\n/m); + assert.equal(code.match(/permissions:/g).length, 2); + assert.match(code, /^ {4}permissions:\n {6}actions: read *\n {6}contents: write *\n {6}pull-requests: write *\n {4}steps:/m); + assert.doesNotMatch(code, /secrets\.|id-token/); + const uses = [...code.matchAll(/uses:\s*(\S+)/g)].map((m) => m[1]); + assert.equal(uses.length, 3); + for (const u of uses) assert.match(u, /^actions\/[\w-]+@[0-9a-f]{40}$/, u); +});