From 6b88af1367ff2958faec12aa661515efc87af891 Mon Sep 17 00:00:00 2001 From: Rigel Date: Mon, 2 Nov 2026 05:29:32 +0000 Subject: [PATCH] feat: visual-diff check draws base and head for UI changes (#74) A pull_request workflow on ubuntu-latest with a read-only token and no secrets. It plans from the merge commit's diff, and when public/ (not public/usage/) or the check itself changed it draws ?demo from the base and the head at 1440x900 and 375x812 with a pinned Playwright, writes a summary, and uploads the shots as the visual-diff artifact. It fails only when the head page errs, and passes when no UI path changed, so it can be a required check. Posting the comment is the next slice. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/visual-diff.yml | 59 +++++++++++++ AGENTS.md | 8 +- scripts/visual-diff.mjs | 142 ++++++++++++++++++++++++++++++ test/visual-diff.test.mjs | 88 ++++++++++++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/visual-diff.yml create mode 100644 scripts/visual-diff.mjs create mode 100644 test/visual-diff.test.mjs diff --git a/.github/workflows/visual-diff.yml b/.github/workflows/visual-diff.yml new file mode 100644 index 0000000..56f917b --- /dev/null +++ b/.github/workflows/visual-diff.yml @@ -0,0 +1,59 @@ +name: Visual diff + +# setkernel-live#74: a pull request that touches the page gets screenshots of its base and its head +# at 1440 x 900 and 375 x 812 (scripts/visual-diff.mjs), kept as the run's visual-diff artifact. +# The job always runs and passes when no UI path changed, so it can be a required check (hq#799): +# no paths filter, which would leave a required check waiting for a run that never starts. +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: visual-diff-${{ github.event.pull_request.number }} + cancel-in-progress: true + +defaults: + run: + shell: bash # -eo pipefail, so a failed git diff fails the plan + +jobs: + # Public repo: a fork's pull request runs here with a read-only token and no secrets, so never a + # self-hosted runner (setkernel-live#59) and never pull_request_target, which would run the + # fork's code with this repo's token. + visual-diff: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # The merge commit is the head; its first parent is the base, so depth 2 holds both. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 2 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + - id: plan + run: git diff --no-renames --name-only HEAD^1 HEAD | node scripts/visual-diff.mjs plan >> "$GITHUB_OUTPUT" + - if: steps.plan.outputs.ui != 'true' + run: echo "No UI path changed, so nothing was drawn." >> "$GITHUB_STEP_SUMMARY" + - if: steps.plan.outputs.ui == 'true' + run: git worktree add --detach "$RUNNER_TEMP/base" HEAD^1 + # Playwright isn't a dependency of this repo: a pinned one outside it, with no install scripts. + - if: steps.plan.outputs.ui == 'true' + run: | + npm install --prefix "$RUNNER_TEMP/pw" --no-save --no-audit --no-fund --ignore-scripts playwright@1.64.1 + "$RUNNER_TEMP/pw/node_modules/.bin/playwright" install --with-deps --only-shell chromium + - if: steps.plan.outputs.ui == 'true' + env: + PLAYWRIGHT_DIR: ${{ runner.temp }}/pw + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: node scripts/visual-diff.mjs render "$RUNNER_TEMP/base" . "$RUNNER_TEMP/visual-diff" >> "$GITHUB_STEP_SUMMARY" + - if: ${{ !cancelled() && steps.plan.outputs.ui == 'true' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: visual-diff + path: ${{ runner.temp }}/visual-diff + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 4775b16..7ff24f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,8 @@ Public repo: everything here is visible to anyone. # the line's columns, Access token check, GitHub receiver, page wording/ # headline/demo/replay, Day's lanes/Now column/key/scrubbing/fitWall, # Buzz alert, page hosts, fonts and contrast, the smoke check, the showcase's -# publisher, the workflow runner guard, and repo names against GitHub's public list, live) +# publisher, the workflow runner guard, the visual-diff check's plan and workflow, +# and repo names against GitHub's public list, live) # worker: npm run test:worker (vitest in workerd: the real Worker and Hub, see test/worker/) # lint: npm run check (node --check on every script) # validate: npm run validate (check + test + test:worker + wrangler deploy --dry-run; what CI runs) @@ -63,6 +64,11 @@ a laptop. pure and tested; `chart.js` draws the SVG. - `scripts/smoke.mjs`: the post-deploy smoke check (no dependencies); waits for `/api/state`'s `version` (the `VERSION` var, set at deploy) to be the deployed commit, then checks each route. +- `scripts/visual-diff.mjs` and `.github/workflows/visual-diff.yml`: the visual-diff check (setkernel-live#74). + On every pull request that touches `public/` (not `public/usage/`), it draws `?demo` from the base and + from the head at 1440x900 and 375x812, on one runner with a pinned Playwright, and keeps the shots + as the run's `visual-diff` artifact. It fails only when the head page errs. It always runs, and + passes with no UI change, so it can be a required check: never give it a `paths:` filter or a job `if:`. - `test/`: `node --test` suites for the pure modules and the page (`day.test.mjs` for Day). - `test/worker/`: vitest suites run in workerd by `@cloudflare/vitest-plugin` (`vitest.config.mjs` reads `wrangler.jsonc`). They fake Access's certs with a throwaway key and drive every route diff --git a/scripts/visual-diff.mjs b/scripts/visual-diff.mjs new file mode 100644 index 0000000..33dcd75 --- /dev/null +++ b/scripts/visual-diff.mjs @@ -0,0 +1,142 @@ +// The visual-diff check (setkernel-live#74). A pull request that touches the page gets it drawn +// twice on the same runner: as its base (the merge commit's first parent) and as its head (the +// merge commit), at 1440 x 900 and 375 x 812, on ?demo in Halifax time at a fixed clock, as +// docs/design/v3/day/built/shoot.mjs does. Each side's public/ is served with its own +// public/_headers. A pull request that touches no UI path draws nothing and passes. +// +// git diff --no-renames --name-only HEAD^1 HEAD | node scripts/visual-diff.mjs plan +// prints ui=true or ui=false +// PLAYWRIGHT_DIR= node scripts/visual-diff.mjs render +// writes /base/*.png, /head/*.png and /visual-diff.json, prints a summary +// +// Exit 1 from render: the head page threw, broke its CSP, asked for anything off its own origin +// or never drew. A visual change is not a failure: the shots are for the reviewers. A failure on +// the base side is only reported, since the base is already merged. +// Playwright isn't a dependency of this repo; .github/workflows/visual-diff.yml installs a pinned +// one outside it, as shoot.mjs's header says. + +import { createServer } from "node:http"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +// What the page is built from: public/, except the private usage page, which this check never +// draws; and this check itself, so a change to it shows that it still draws. +const UI = [/^public\//, /^scripts\/visual-diff\.mjs$/, /^\.github\/workflows\/visual-diff\.yml$/]; +const NOT_UI = [/^public\/usage\//]; +export const isUiPath = (p) => UI.some((r) => r.test(p)) && !NOT_UI.some((r) => r.test(p)); +export const uiChanged = (paths) => paths.some(isUiPath); + +// [name, width, height, query] +export const SHOTS = [ + ["demo-1440x900", 1440, 900, "?demo"], + ["demo-375x812", 375, 812, "?demo"], +]; +export const NOW = Date.UTC(2026, 9, 14, 19, 40, 0); // 16:40 ADT, shoot.mjs's clock + +const status = (s) => + s.head.errors.length ? `head failed: ${s.head.errors.join("; ")}` + : s.base.errors.length ? `base failed: ${s.base.errors.join("; ")}` + : s.changed ? "changed" : "same"; + +// The run's summary: one row per shot, base against head. +export function summary(shots) { + const rows = shots.map((s) => `| ${s.name} | ${s.width} x ${s.height} | \`${s.query}\` | ${status(s)} |`); + const changed = shots.filter((s) => s.changed).length; + const failed = shots.filter((s) => s.head.errors.length).length; + return [ + "### Visual diff", + failed ? `The head failed to draw cleanly in ${failed} of ${shots.length} shots, so this check fails.` + : changed ? `${changed} of ${shots.length} shots changed; both sides are in the run's \`visual-diff\` artifact.` + : `No shot changed (${shots.length} drawn).`, + "", + "| Shot | Size | Page | Base to head |", + "| --- | --- | --- | --- |", + ...rows, + "", + ].join("\n"); +} + +// A static server over one checkout's public/, with the headers its public/_headers sets for /*. +const TYPES = { html: "text/html", js: "text/javascript", css: "text/css", svg: "image/svg+xml", woff2: "font/woff2", png: "image/png" }; +async function serve(pub) { + const headers = Object.fromEntries((await readFile(join(pub, "_headers"), "utf8")).split("\n") + .filter((l) => /^\s+\S+:/.test(l)).map((l) => [l.trim().slice(0, l.trim().indexOf(":")), l.slice(l.indexOf(":") + 1).trim()])); + const server = createServer(async (req, res) => { + const path = new URL(req.url, "http://x").pathname.replace(/^\/$/, "/index.html"); + try { + const body = await readFile(join(pub, path)); + res.writeHead(200, { ...headers, "Content-Type": TYPES[path.split(".").pop()] ?? "application/octet-stream" }); + res.end(body); + } catch { + res.writeHead(404).end(); + } + }).listen(0, "127.0.0.1"); + await new Promise((ok) => server.once("listening", ok)); + return { origin: `http://127.0.0.1:${server.address().port}`, close: () => server.close() }; +} + +async function shoot(browser, origin, [, width, height, query], file) { + const errors = []; + const ctx = await browser.newContext({ viewport: { width, height }, timezoneId: "America/Halifax", locale: "en-CA" }); + try { + const p = await ctx.newPage(); + await p.clock.setFixedTime(NOW); + p.on("pageerror", (e) => errors.push(e.message)); + p.on("console", (m) => { if (m.type() === "error" || /Content Security Policy/.test(m.text())) errors.push(m.text()); }); + p.on("request", (r) => { if (!r.url().startsWith(origin)) errors.push(`network: ${r.url()}`); }); + await p.goto(`${origin}/${query}`); + await p.waitForFunction(() => document.querySelector("#plot svg") && document.fonts.status === "loaded", null, { timeout: 15_000 }); + await p.waitForTimeout(300); + await p.screenshot({ path: file }); + return { errors, png: await readFile(file) }; + } catch (e) { + return { errors: [...errors, e.message.split("\n")[0]], png: null }; + } finally { + await ctx.close(); + } +} + +export async function render(baseDir, headDir, outDir) { + const dir = process.env.PLAYWRIGHT_DIR; + if (!dir) throw new Error("set PLAYWRIGHT_DIR to a folder with playwright installed"); + const { chromium } = createRequire(`${resolve(dir)}/`)("playwright"); + const sides = { base: baseDir, head: headDir }; + const shots = SHOTS.map(([name, width, height, query]) => ({ name, width, height, query })); + const browser = await chromium.launch(); + try { + for (const [side, root] of Object.entries(sides)) { + await mkdir(join(outDir, side), { recursive: true }); + const server = await serve(join(root, "public")); + try { + for (const [i, shot] of SHOTS.entries()) + shots[i][side] = await shoot(browser, server.origin, shot, join(outDir, side, `${shot[0]}.png`)); + } finally { + server.close(); + } + } + } finally { + await browser.close(); + } + for (const s of shots) s.changed = !!(s.base.png && s.head.png) && !s.base.png.equals(s.head.png); + return shots.map(({ base, head, ...s }) => ({ ...s, base: { errors: base.errors }, head: { errors: head.errors } })); +} + +// Not import.meta.main: that needs Node 24.2 (scripts/smoke.mjs says the same). +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [cmd, ...args] = process.argv.slice(2); + if (cmd === "plan" && !args.length) { + console.log(`ui=${uiChanged(readFileSync(0, "utf8").split("\n").map((l) => l.trim()).filter(Boolean))}`); + } else if (cmd === "render" && args.length === 3) { + const shots = await render(...args); + const pr = { number: Number(process.env.PR_NUMBER) || null, head: process.env.HEAD_SHA || null }; + await writeFile(join(args[2], "visual-diff.json"), `${JSON.stringify({ pr, shots }, null, 2)}\n`); + console.log(summary(shots)); + if (shots.some((s) => s.head.errors.length)) process.exitCode = 1; + } else { + console.error("usage: visual-diff.mjs plan < paths | visual-diff.mjs render "); + process.exitCode = 2; + } +} diff --git a/test/visual-diff.test.mjs b/test/visual-diff.test.mjs new file mode 100644 index 0000000..05f395b --- /dev/null +++ b/test/visual-diff.test.mjs @@ -0,0 +1,88 @@ +// The visual-diff check (setkernel-live#74): which paths count as UI, its plan step, its summary, +// and the workflow's shape: a required check that a fork's pull request can run. The drawing itself +// needs Playwright, which isn't a dependency; the workflow runs it. Run: npm test +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { SHOTS, isUiPath, summary, uiChanged } from "../scripts/visual-diff.mjs"; + +const script = fileURLToPath(new URL("../scripts/visual-diff.mjs", import.meta.url)); +const plan = (paths) => spawnSync(process.execPath, [script, "plan"], { input: paths.join("\n"), encoding: "utf8" }); +const workflow = readFileSync(new URL("../.github/workflows/visual-diff.yml", import.meta.url), "utf8"); +const code = workflow.replace(/#.*$/gm, ""); + +test("the page's files are UI paths; the usage page, the Worker and docs are not", () => { + for (const p of ["public/index.html", "public/app.js", "public/lib/day.js", "public/fonts/x.woff2", "public/_headers", + "scripts/visual-diff.mjs", ".github/workflows/visual-diff.yml"]) assert.ok(isUiPath(p), p); + for (const p of ["public/usage/index.html", "public/usage/table.js", "src/worker.js", "scripts/smoke.mjs", "README.md", + "docs/design/v3/day/index.html", "test/page.test.mjs", "wrangler.jsonc", ".github/workflows/ci.yml", "xpublic/app.js"]) + assert.ok(!isUiPath(p), p); + assert.equal(uiChanged([]), false); + assert.equal(uiChanged(["README.md", "public/style.css"]), true); +}); + +test("plan says ui=false, and exits 0, when no UI path changed", () => { + for (const paths of [[], ["src/worker.js", "public/usage/app.js", "AGENTS.md"]]) { + const r = plan(paths); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, "ui=false\n"); + } + const r = plan(["src/worker.js", " public/app.js ", ""]); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, "ui=true\n"); +}); + +test("bad usage exits 2", () => { + assert.equal(spawnSync(process.execPath, [script], { encoding: "utf8" }).status, 2); + assert.equal(spawnSync(process.execPath, [script, "render", "a"], { encoding: "utf8" }).status, 2); +}); + +test("it draws ?demo at 1440 x 900 and 375 x 812", () => { + assert.deepEqual(SHOTS.map(([, w, h, q]) => `${w}x${h}${q}`), ["1440x900?demo", "375x812?demo"]); + assert.equal(new Set(SHOTS.map(([n]) => n)).size, SHOTS.length); +}); + +test("the summary has a row per shot and says what changed or failed", () => { + const ok = { errors: [] }; + const shot = (name, changed, head = ok, base = ok) => ({ name, width: 375, height: 812, query: "?demo", changed, base, head }); + const same = summary([shot("a", false), shot("b", false)]); + assert.match(same, /No shot changed \(2 drawn\)/); + assert.match(same, /\| a \| 375 x 812 \| `\?demo` \| same \|/); + const some = summary([shot("a", true), shot("b", false, ok, { errors: ["boom"] })]); + assert.match(some, /1 of 2 shots changed/); + assert.match(some, /\| a \|.*\| changed \|/); + assert.match(some, /\| b \|.*\| base failed: boom \|/); + const bad = summary([shot("a", false, { errors: ["network: https://x.example/"] })]); + assert.match(bad, /The head failed to draw cleanly in 1 of 1 shots/); + assert.match(bad, /head failed: network: https:\/\/x\.example\//); +}); + +test("the workflow runs on every pull request, with no paths filter and no job condition", () => { + assert.match(code, /^on:\n {2}pull_request:\n/m); + assert.doesNotMatch(code, /pull_request_target|workflow_run|paths(-ignore)?:/); + assert.match(code, /^ {2}visual-diff:\n {4}runs-on: ubuntu-latest\n/m); + assert.doesNotMatch(code, /^ {4}if:/m, "a job-level if would skip the required check"); + // Every step past the plan that draws or installs waits for ui=true. + const steps = code.split(/^ {6}- /m).slice(1); + const drawing = steps.filter((s) => /playwright|render|worktree|upload-artifact/.test(s)); + assert.equal(drawing.length, 4); + for (const s of drawing) assert.match(s, /steps\.plan\.outputs\.ui == 'true'/, s); +}); + +test("the workflow needs no secrets and no write access", () => { + assert.doesNotMatch(code, /secrets\.|github\.token|GITHUB_TOKEN|id-token/); + assert.equal(code.match(/permissions:/g).length, 1, "one permissions block, at the top"); + assert.doesNotMatch(code, /: *write\b/); + assert.match(code, /^permissions:\n {2}contents: read\n\n/m); + assert.match(code, /persist-credentials: false/); +}); + +test("the workflow pins its actions and Playwright", () => { + const uses = [...code.matchAll(/uses:\s*(\S+)/g)].map((m) => m[1]); + assert.equal(uses.length, 3); + for (const u of uses) assert.match(u, /^actions\/[\w-]+@[0-9a-f]{40}$/, u); + assert.match(code, /playwright@\d+\.\d+\.\d+ *\n/); + assert.match(code, /--ignore-scripts/); +});