Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions .github/workflows/visual-diff.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Visual diff

# setkernel-live#74: a pull request that touches the page gets screenshots of its base and its head
# at 1440 x 900 and 375 x 812 (scripts/visual-diff.mjs), kept as the run's visual-diff artifact.
# The job always runs and passes when no UI path changed, so it can be a required check (hq#799):
# no paths filter, which would leave a required check waiting for a run that never starts.
on:
pull_request:

permissions:
contents: read

concurrency:
group: visual-diff-${{ github.event.pull_request.number }}
cancel-in-progress: true

defaults:
run:
shell: bash # -eo pipefail, so a failed git diff fails the plan

jobs:
# Public repo: a fork's pull request runs here with a read-only token and no secrets, so never a
# self-hosted runner (setkernel-live#59) and never pull_request_target, which would run the
# fork's code with this repo's token.
visual-diff:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
# The merge commit is the head; its first parent is the base, so depth 2 holds both.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 2
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version-file: .nvmrc
- id: plan
run: git diff --no-renames --name-only HEAD^1 HEAD | node scripts/visual-diff.mjs plan >> "$GITHUB_OUTPUT"
- if: steps.plan.outputs.ui != 'true'
run: echo "No UI path changed, so nothing was drawn." >> "$GITHUB_STEP_SUMMARY"
- if: steps.plan.outputs.ui == 'true'
run: git worktree add --detach "$RUNNER_TEMP/base" HEAD^1
# Playwright isn't a dependency of this repo: a pinned one outside it, with no install scripts.
- if: steps.plan.outputs.ui == 'true'
run: |
npm install --prefix "$RUNNER_TEMP/pw" --no-save --no-audit --no-fund --ignore-scripts playwright@1.64.1
"$RUNNER_TEMP/pw/node_modules/.bin/playwright" install --with-deps --only-shell chromium
- if: steps.plan.outputs.ui == 'true'
env:
PLAYWRIGHT_DIR: ${{ runner.temp }}/pw
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node scripts/visual-diff.mjs render "$RUNNER_TEMP/base" . "$RUNNER_TEMP/visual-diff" >> "$GITHUB_STEP_SUMMARY"
- if: ${{ !cancelled() && steps.plan.outputs.ui == 'true' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: visual-diff
path: ${{ runner.temp }}/visual-diff
retention-days: 14
8 changes: 7 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ Public repo: everything here is visible to anyone.
# the line's columns, Access token check, GitHub receiver, page wording/
# headline/demo/replay, Day's lanes/Now column/key/scrubbing/fitWall,
# Buzz alert, page hosts, fonts and contrast, the smoke check, the showcase's
# publisher, the workflow runner guard, and repo names against GitHub's public list, live)
# publisher, the workflow runner guard, the visual-diff check's plan and workflow,
# and repo names against GitHub's public list, live)
# worker: npm run test:worker (vitest in workerd: the real Worker and Hub, see test/worker/)
# lint: npm run check (node --check on every script)
# validate: npm run validate (check + test + test:worker + wrangler deploy --dry-run; what CI runs)
Expand Down Expand Up @@ -63,6 +64,11 @@ a laptop.
pure and tested; `chart.js` draws the SVG.
- `scripts/smoke.mjs`: the post-deploy smoke check (no dependencies); waits for `/api/state`'s
`version` (the `VERSION` var, set at deploy) to be the deployed commit, then checks each route.
- `scripts/visual-diff.mjs` and `.github/workflows/visual-diff.yml`: the visual-diff check (setkernel-live#74).
On every pull request that touches `public/` (not `public/usage/`), it draws `?demo` from the base and
from the head at 1440x900 and 375x812, on one runner with a pinned Playwright, and keeps the shots
as the run's `visual-diff` artifact. It fails only when the head page errs. It always runs, and
passes with no UI change, so it can be a required check: never give it a `paths:` filter or a job `if:`.
- `test/`: `node --test` suites for the pure modules and the page (`day.test.mjs` for Day).
- `test/worker/`: vitest suites run in workerd by `@cloudflare/vitest-plugin` (`vitest.config.mjs`
reads `wrangler.jsonc`). They fake Access's certs with a throwaway key and drive every route
Expand Down
142 changes: 142 additions & 0 deletions scripts/visual-diff.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// The visual-diff check (setkernel-live#74). A pull request that touches the page gets it drawn
// twice on the same runner: as its base (the merge commit's first parent) and as its head (the
// merge commit), at 1440 x 900 and 375 x 812, on ?demo in Halifax time at a fixed clock, as
// docs/design/v3/day/built/shoot.mjs does. Each side's public/ is served with its own
// public/_headers. A pull request that touches no UI path draws nothing and passes.
//
// git diff --no-renames --name-only HEAD^1 HEAD | node scripts/visual-diff.mjs plan
// prints ui=true or ui=false
// PLAYWRIGHT_DIR=<dir> node scripts/visual-diff.mjs render <base checkout> <head checkout> <out>
// writes <out>/base/*.png, <out>/head/*.png and <out>/visual-diff.json, prints a summary
//
// Exit 1 from render: the head page threw, broke its CSP, asked for anything off its own origin
// or never drew. A visual change is not a failure: the shots are for the reviewers. A failure on
// the base side is only reported, since the base is already merged.
// Playwright isn't a dependency of this repo; .github/workflows/visual-diff.yml installs a pinned
// one outside it, as shoot.mjs's header says.

import { createServer } from "node:http";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { readFileSync } from "node:fs";
import { createRequire } from "node:module";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";

// What the page is built from: public/, except the private usage page, which this check never
// draws; and this check itself, so a change to it shows that it still draws.
const UI = [/^public\//, /^scripts\/visual-diff\.mjs$/, /^\.github\/workflows\/visual-diff\.yml$/];
const NOT_UI = [/^public\/usage\//];
export const isUiPath = (p) => UI.some((r) => r.test(p)) && !NOT_UI.some((r) => r.test(p));
export const uiChanged = (paths) => paths.some(isUiPath);

// [name, width, height, query]
export const SHOTS = [
["demo-1440x900", 1440, 900, "?demo"],
["demo-375x812", 375, 812, "?demo"],
];
export const NOW = Date.UTC(2026, 9, 14, 19, 40, 0); // 16:40 ADT, shoot.mjs's clock

const status = (s) =>
s.head.errors.length ? `head failed: ${s.head.errors.join("; ")}`
: s.base.errors.length ? `base failed: ${s.base.errors.join("; ")}`
: s.changed ? "changed" : "same";

// The run's summary: one row per shot, base against head.
export function summary(shots) {
const rows = shots.map((s) => `| ${s.name} | ${s.width} x ${s.height} | \`${s.query}\` | ${status(s)} |`);
const changed = shots.filter((s) => s.changed).length;
const failed = shots.filter((s) => s.head.errors.length).length;
return [
"### Visual diff",
failed ? `The head failed to draw cleanly in ${failed} of ${shots.length} shots, so this check fails.`
: changed ? `${changed} of ${shots.length} shots changed; both sides are in the run's \`visual-diff\` artifact.`
: `No shot changed (${shots.length} drawn).`,
"",
"| Shot | Size | Page | Base to head |",
"| --- | --- | --- | --- |",
...rows,
"",
].join("\n");
}

// A static server over one checkout's public/, with the headers its public/_headers sets for /*.
const TYPES = { html: "text/html", js: "text/javascript", css: "text/css", svg: "image/svg+xml", woff2: "font/woff2", png: "image/png" };
async function serve(pub) {
const headers = Object.fromEntries((await readFile(join(pub, "_headers"), "utf8")).split("\n")
.filter((l) => /^\s+\S+:/.test(l)).map((l) => [l.trim().slice(0, l.trim().indexOf(":")), l.slice(l.indexOf(":") + 1).trim()]));
const server = createServer(async (req, res) => {
const path = new URL(req.url, "http://x").pathname.replace(/^\/$/, "/index.html");
try {
const body = await readFile(join(pub, path));
res.writeHead(200, { ...headers, "Content-Type": TYPES[path.split(".").pop()] ?? "application/octet-stream" });
res.end(body);
} catch {
res.writeHead(404).end();
}
}).listen(0, "127.0.0.1");
await new Promise((ok) => server.once("listening", ok));
return { origin: `http://127.0.0.1:${server.address().port}`, close: () => server.close() };
}

async function shoot(browser, origin, [, width, height, query], file) {
const errors = [];
const ctx = await browser.newContext({ viewport: { width, height }, timezoneId: "America/Halifax", locale: "en-CA" });
try {
const p = await ctx.newPage();
await p.clock.setFixedTime(NOW);
p.on("pageerror", (e) => errors.push(e.message));
p.on("console", (m) => { if (m.type() === "error" || /Content Security Policy/.test(m.text())) errors.push(m.text()); });
p.on("request", (r) => { if (!r.url().startsWith(origin)) errors.push(`network: ${r.url()}`); });
await p.goto(`${origin}/${query}`);
await p.waitForFunction(() => document.querySelector("#plot svg") && document.fonts.status === "loaded", null, { timeout: 15_000 });
await p.waitForTimeout(300);
await p.screenshot({ path: file });
return { errors, png: await readFile(file) };
} catch (e) {
return { errors: [...errors, e.message.split("\n")[0]], png: null };
} finally {
await ctx.close();
}
}

export async function render(baseDir, headDir, outDir) {
const dir = process.env.PLAYWRIGHT_DIR;
if (!dir) throw new Error("set PLAYWRIGHT_DIR to a folder with playwright installed");
const { chromium } = createRequire(`${resolve(dir)}/`)("playwright");
const sides = { base: baseDir, head: headDir };
const shots = SHOTS.map(([name, width, height, query]) => ({ name, width, height, query }));
const browser = await chromium.launch();
try {
for (const [side, root] of Object.entries(sides)) {
await mkdir(join(outDir, side), { recursive: true });
const server = await serve(join(root, "public"));
try {
for (const [i, shot] of SHOTS.entries())
shots[i][side] = await shoot(browser, server.origin, shot, join(outDir, side, `${shot[0]}.png`));
} finally {
server.close();
}
}
} finally {
await browser.close();
}
for (const s of shots) s.changed = !!(s.base.png && s.head.png) && !s.base.png.equals(s.head.png);
return shots.map(({ base, head, ...s }) => ({ ...s, base: { errors: base.errors }, head: { errors: head.errors } }));
}

// Not import.meta.main: that needs Node 24.2 (scripts/smoke.mjs says the same).
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
const [cmd, ...args] = process.argv.slice(2);
if (cmd === "plan" && !args.length) {
console.log(`ui=${uiChanged(readFileSync(0, "utf8").split("\n").map((l) => l.trim()).filter(Boolean))}`);
} else if (cmd === "render" && args.length === 3) {
const shots = await render(...args);
const pr = { number: Number(process.env.PR_NUMBER) || null, head: process.env.HEAD_SHA || null };
await writeFile(join(args[2], "visual-diff.json"), `${JSON.stringify({ pr, shots }, null, 2)}\n`);
console.log(summary(shots));
if (shots.some((s) => s.head.errors.length)) process.exitCode = 1;
} else {
console.error("usage: visual-diff.mjs plan < paths | visual-diff.mjs render <base> <head> <out>");
process.exitCode = 2;
}
}
88 changes: 88 additions & 0 deletions test/visual-diff.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// The visual-diff check (setkernel-live#74): which paths count as UI, its plan step, its summary,
// and the workflow's shape: a required check that a fork's pull request can run. The drawing itself
// needs Playwright, which isn't a dependency; the workflow runs it. Run: npm test
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { SHOTS, isUiPath, summary, uiChanged } from "../scripts/visual-diff.mjs";

const script = fileURLToPath(new URL("../scripts/visual-diff.mjs", import.meta.url));
const plan = (paths) => spawnSync(process.execPath, [script, "plan"], { input: paths.join("\n"), encoding: "utf8" });
const workflow = readFileSync(new URL("../.github/workflows/visual-diff.yml", import.meta.url), "utf8");
const code = workflow.replace(/#.*$/gm, "");

test("the page's files are UI paths; the usage page, the Worker and docs are not", () => {
for (const p of ["public/index.html", "public/app.js", "public/lib/day.js", "public/fonts/x.woff2", "public/_headers",
"scripts/visual-diff.mjs", ".github/workflows/visual-diff.yml"]) assert.ok(isUiPath(p), p);
for (const p of ["public/usage/index.html", "public/usage/table.js", "src/worker.js", "scripts/smoke.mjs", "README.md",
"docs/design/v3/day/index.html", "test/page.test.mjs", "wrangler.jsonc", ".github/workflows/ci.yml", "xpublic/app.js"])
assert.ok(!isUiPath(p), p);
assert.equal(uiChanged([]), false);
assert.equal(uiChanged(["README.md", "public/style.css"]), true);
});

test("plan says ui=false, and exits 0, when no UI path changed", () => {
for (const paths of [[], ["src/worker.js", "public/usage/app.js", "AGENTS.md"]]) {
const r = plan(paths);
assert.equal(r.status, 0, r.stderr);
assert.equal(r.stdout, "ui=false\n");
}
const r = plan(["src/worker.js", " public/app.js ", ""]);
assert.equal(r.status, 0, r.stderr);
assert.equal(r.stdout, "ui=true\n");
});

test("bad usage exits 2", () => {
assert.equal(spawnSync(process.execPath, [script], { encoding: "utf8" }).status, 2);
assert.equal(spawnSync(process.execPath, [script, "render", "a"], { encoding: "utf8" }).status, 2);
});

test("it draws ?demo at 1440 x 900 and 375 x 812", () => {
assert.deepEqual(SHOTS.map(([, w, h, q]) => `${w}x${h}${q}`), ["1440x900?demo", "375x812?demo"]);
assert.equal(new Set(SHOTS.map(([n]) => n)).size, SHOTS.length);
});

test("the summary has a row per shot and says what changed or failed", () => {
const ok = { errors: [] };
const shot = (name, changed, head = ok, base = ok) => ({ name, width: 375, height: 812, query: "?demo", changed, base, head });
const same = summary([shot("a", false), shot("b", false)]);
assert.match(same, /No shot changed \(2 drawn\)/);
assert.match(same, /\| a \| 375 x 812 \| `\?demo` \| same \|/);
const some = summary([shot("a", true), shot("b", false, ok, { errors: ["boom"] })]);
assert.match(some, /1 of 2 shots changed/);
assert.match(some, /\| a \|.*\| changed \|/);
assert.match(some, /\| b \|.*\| base failed: boom \|/);
const bad = summary([shot("a", false, { errors: ["network: https://x.example/"] })]);
assert.match(bad, /The head failed to draw cleanly in 1 of 1 shots/);
assert.match(bad, /head failed: network: https:\/\/x\.example\//);
});

test("the workflow runs on every pull request, with no paths filter and no job condition", () => {
assert.match(code, /^on:\n {2}pull_request:\n/m);
assert.doesNotMatch(code, /pull_request_target|workflow_run|paths(-ignore)?:/);
assert.match(code, /^ {2}visual-diff:\n {4}runs-on: ubuntu-latest\n/m);
assert.doesNotMatch(code, /^ {4}if:/m, "a job-level if would skip the required check");
// Every step past the plan that draws or installs waits for ui=true.
const steps = code.split(/^ {6}- /m).slice(1);
const drawing = steps.filter((s) => /playwright|render|worktree|upload-artifact/.test(s));
assert.equal(drawing.length, 4);
for (const s of drawing) assert.match(s, /steps\.plan\.outputs\.ui == 'true'/, s);
});

test("the workflow needs no secrets and no write access", () => {
assert.doesNotMatch(code, /secrets\.|github\.token|GITHUB_TOKEN|id-token/);
assert.equal(code.match(/permissions:/g).length, 1, "one permissions block, at the top");
assert.doesNotMatch(code, /: *write\b/);
assert.match(code, /^permissions:\n {2}contents: read\n\n/m);
assert.match(code, /persist-credentials: false/);
});

test("the workflow pins its actions and Playwright", () => {
const uses = [...code.matchAll(/uses:\s*(\S+)/g)].map((m) => m[1]);
assert.equal(uses.length, 3);
for (const u of uses) assert.match(u, /^actions\/[\w-]+@[0-9a-f]{40}$/, u);
assert.match(code, /playwright@\d+\.\d+\.\d+ *\n/);
assert.match(code, /--ignore-scripts/);
});
Loading