From 79a1b7a066ded2b71410b2e232ca249181ab6710 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:38:04 +0000 Subject: [PATCH 1/6] dev2-edge: pit.world and the Moshpit names on Caddy, every other site on nginx nginx's stream module takes :443 and routes by TLS server name (ssl_preread): pit.world, *.pit.world and the Moshpit names in dev2/edge/edge.json go to Caddy on 127.0.0.1:8453, everything else to nginx's https servers on 127.0.0.1:8444, both with a PROXY protocol header so client addresses survive. - dev2/dev2-edge: render / apply / route on|off / rollback / sweep / status - dev2/edge/edge-box: the box side; apply backs up /etc/nginx, sweeps every name twice, checks Caddy answers like nginx on its own port before any traffic moves, switches with one nginx reload, sweeps again and rolls back on any change - caddy-edge-certs (+ .path/.timer): Caddy loads bundles of the certificates certbot, pit-proxy-cert and moshpit-renew already renew; Caddy issues nothing - nginx-edge-listen (+ nginx.service drop-in, .path): rewrites any `listen 443` a tool writes later (install.sh, setup-origin.sh, certbot --nginx), which would otherwise pass nginx -t and then silently fail to bind on reload - dev2-site vhost renders the 8444 form when the edge is on - test/dev2-edge.test.ts Rehearsed on dev3 against a copy of dev2's nginx config: 287 names same status before/after, SNI routing, client IP in nginx and Caddy upstreams, websockets, http->https, no dropped requests across the switch and the rollback, renewal pickup, stray listen 443 repair, rollback restores /etc/nginx byte for byte. Co-Authored-By: Claude Opus 5.5 --- dev2/README.md | 53 +++ dev2/dev2-edge | 360 +++++++++++++++ dev2/dev2-site | 14 + dev2/edge/caddy-edge-certs | 77 ++++ dev2/edge/edge-box | 417 ++++++++++++++++++ dev2/edge/edge-sweep | 42 ++ dev2/edge/edge.json | 33 ++ dev2/edge/nginx-edge-listen | 163 +++++++ dev2/edge/systemd/caddy-edge-certs.path | 15 + dev2/edge/systemd/caddy-edge-certs.service | 7 + dev2/edge/systemd/caddy-edge-certs.timer | 12 + dev2/edge/systemd/caddy.service | 24 + dev2/edge/systemd/nginx-edge-listen.path | 14 + dev2/edge/systemd/nginx-edge-listen.service | 7 + .../systemd/nginx.service.d-dev2-edge.conf | 12 + test/dev2-edge.test.ts | 239 ++++++++++ 16 files changed, 1489 insertions(+) create mode 100755 dev2/dev2-edge create mode 100755 dev2/edge/caddy-edge-certs create mode 100755 dev2/edge/edge-box create mode 100755 dev2/edge/edge-sweep create mode 100644 dev2/edge/edge.json create mode 100755 dev2/edge/nginx-edge-listen create mode 100644 dev2/edge/systemd/caddy-edge-certs.path create mode 100644 dev2/edge/systemd/caddy-edge-certs.service create mode 100644 dev2/edge/systemd/caddy-edge-certs.timer create mode 100644 dev2/edge/systemd/caddy.service create mode 100644 dev2/edge/systemd/nginx-edge-listen.path create mode 100644 dev2/edge/systemd/nginx-edge-listen.service create mode 100644 dev2/edge/systemd/nginx.service.d-dev2-edge.conf create mode 100644 test/dev2-edge.test.ts diff --git a/dev2/README.md b/dev2/README.md index 2fb7879..f8ee3f1 100644 --- a/dev2/README.md +++ b/dev2/README.md @@ -223,3 +223,56 @@ deploy-poll.sh --dry-run what one pass would deploy Logs: `~/.local/state/deploy-poll/poll.log` and `.log`. + +## The edge: pit.world on Caddy, everything else on nginx (`dev2-edge`) + +Since 2026-10-19 nginx's `stream {}` module owns :443 on dev2 and routes each +connection by TLS server name (`ssl_preread`; TLS is not terminated there): + +| name | goes to | +| --- | --- | +| `pit.world`, `www.pit.world`, `*.pit.world` (gateways, `bbs.`/`social.`), the Moshpit names in `edge/edge.json` | Caddy, `127.0.0.1:8453` | +| everything else (and `live.streaming.pit.world`, see `keep_on_nginx`) | nginx's own https servers, `127.0.0.1:8444` | + +Both hops carry a PROXY protocol header, so nginx (`real_ip_header proxy_protocol`, +`conf.d/00-edge-realip.conf`) and Caddy (`proxy_protocol` listener wrapper) see the +visitor's address; rate limits, logs and ThreatCrush are unchanged. Port 80 is not +touched: nginx still answers ACME and http->https for every name. UDP 443 (HTTP/3) is +out of scope. + +``` +dev2-edge render [--out DIR] [--off] the box files for edge.json (nothing touches dev2) +dev2-edge apply --yes backup, sweep, Caddy up + checked on its own port, nginx switched, sweep again +dev2-edge route off|on every name back to nginx (or to Caddy again): one nginx reload +dev2-edge rollback [TARBALL] restore /etc/nginx from apply's backup (~anthony/backups/nginx-.tgz), stop Caddy +dev2-edge sweep | status +``` + +`apply` (via `edge/edge-box` on the box) takes the status of `https:///` (verified +and `-k`) and `http:///` for every name nginx and Caddy serve, twice (names that +differ between the two are set aside as flaky), starts Caddy and checks it answers each +routed name exactly as nginx did before any visitor reaches it, switches nginx with one +reload (the :443 socket passes from the http servers to the stream without a dropped +connection), sweeps again, and rolls everything back if any name's status changed. + +**Certificates.** Caddy issues nothing (`auto_https off`). `caddy-edge-certs` copies +what the existing renewals keep current into `/etc/caddy/edge-certs/.pem` +(root:caddy 0640): certbot for `pit.world`, `pit-proxy-cert` for each `.pit.world`, +`moshpit-renew` for the Moshpit names. `caddy-edge-certs.path` runs it within a second +of any change to those directories (daily timer as a backstop) and reloads Caddy only +when a bundle changed. A new `.pit.world` therefore still costs one Let's Encrypt +certificate, issued by `pit-proxy-cert` on the name's first plain-http visit, as before. + +**`listen 443` is now a bug, and is fixed for you.** With the stream on :443 an http +server's `listen 443` passes `nginx -t`, but the reload cannot bind, so nginx keeps the +old configuration without saying so (a restart fails outright). OpenInstall's +`bin/install.sh`, moshpit-proxy's `setup-origin.sh` (every Moshpit renewal) and +`certbot --nginx` still write it. `nginx-edge-listen` rewrites such lines to +`listen 127.0.0.1:8444 ssl proxy_protocol;` (and drops `[::]:443`): it runs before every +nginx start and reload (`nginx.service.d/dev2-edge.conf`) and whenever a vhost file +changes (`nginx-edge-listen.path`). `dev2-site vhost` renders the 8444 form itself when +the edge is on. `nginx-edge-listen --check` lists anything left. + +**Cloudflare.** One `real_ip_header` per level: `conf.d/cloudflare-realip.conf`'s +`CF-Connecting-IP` line is commented out. No dev2 name was proxied by Cloudflare on +2026-10-19; one that is would log Cloudflare's edge address again. diff --git a/dev2/dev2-edge b/dev2/dev2-edge new file mode 100755 index 0000000..15f71ed --- /dev/null +++ b/dev2/dev2-edge @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +""" +dev2-edge: pit.world and the Moshpit names on Caddy, every other dev2 site on nginx. + + dev2-edge render [--out DIR] [--off] write the box files for edge.json into DIR (nothing touches dev2) + dev2-edge apply --yes cut over: backup, baseline sweep, install, switch, sweep again; + any site whose status changed rolls the whole box back + dev2-edge route on|off point the routed names at Caddy (on) or back at nginx (off), + one nginx reload, no other change: the fast way back + dev2-edge rollback [TARBALL] restore /etc/nginx from the apply's backup, stop Caddy + dev2-edge sweep status of every name on dev2 (https verified, https -k, http) + dev2-edge status what is live: routes, Caddy, certificates, listeners + +How it works (agreed with Anthony 2026-10-19, "pit.world first"): + + visitor :443 -> nginx stream {} (ssl_preread, SNI only, TLS not terminated) + pit.world, *.pit.world, Moshpit names in edge.json -> 127.0.0.1:8453 Caddy + everything else -> 127.0.0.1:8444 nginx http + both hops carry a PROXY protocol header, so nginx ($remote_addr via real_ip_header + proxy_protocol) and Caddy (proxy_protocol listener wrapper) see the visitor's address. + Port 80 is untouched: nginx keeps answering ACME and http->https for every name. + +Certificates are never issued by Caddy (auto_https off). caddy-edge-certs copies what +certbot, pit-proxy-cert and moshpit-renew already keep current into bundles Caddy +loads, and reloads Caddy when one changes (a .path unit on the source directories, +plus a daily timer). + +nginx-edge-listen rewrites any `listen 443` a tool writes later (OpenInstall's +bin/install.sh, setup-origin.sh on every Moshpit renewal, certbot --nginx) to the +127.0.0.1:8444 backend; it runs before every nginx start and reload and whenever a +vhost file changes. Without it, such a line passes `nginx -t`, the reload cannot bind +:443 and nginx silently keeps the old configuration (a restart fails outright). + +Run from the box that has root ssh to dev2 (DEV2_HOST). dev2 is production. +""" +import argparse +import datetime as dt +import hashlib +import json +import os +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request + +HERE = os.path.dirname(os.path.realpath(__file__)) +EDGE = os.path.join(HERE, "edge") +SPEC = os.path.join(EDGE, "edge.json") +DEV2_HOST = os.environ.get("DEV2_HOST", "dev2.profullstack.com") +SSH = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=25"] +CACHE = os.path.expanduser(os.environ.get("DEV2_EDGE_CACHE", "~/.cache/dev2-edge")) +HEADER = "# managed by cli-tools/dev2/dev2-edge from dev2/edge/edge.json: do not hand-edit, re-run `dev2-edge apply`\n" + + +def die(msg, code=1): + print(f"ERROR: {msg}", file=sys.stderr, flush=True) + sys.exit(code) + + +def log(msg): + print(f"\n===> {msg}", file=sys.stderr, flush=True) + + +def spec(path=SPEC): + with open(path) as f: + s = json.load(f) + hosts = [h for site in s["sites"] for h in site["hosts"]] + if len(hosts) != len(set(hosts)): + die("edge.json: a host is listed twice") + for site in s["sites"]: + kinds = [k for k in ("proxy", "root", "redirect") if k in site] + if len(kinds) != 1: + die(f"edge.json: {site['hosts']} needs exactly one of proxy, root, redirect") + return s + + +# ------------------------------------------------------------------ rendering + +def stream_pattern(host): + """A Caddy host as an nginx `map ... hostnames` key: nginx's *.x covers any depth.""" + if host.startswith("*."): + return "*." + host.lstrip("*.") + return host + + +def render_stream(s, on=True): + p = s["ports"] + caddy, nginx = f"127.0.0.1:{p['caddy']}", f"127.0.0.1:{p['nginx']}" + rows = [] + if on: + seen = set() + for site in s["sites"]: + for h in site["hosts"]: + k = stream_pattern(h) + if k not in seen: + seen.add(k) + rows.append(f" {k:<32} {caddy};") + for h, why in s.get("keep_on_nginx", {}).items(): + if h.startswith("//"): + continue + rows.append(f" {h:<32} {nginx}; # {why[:90]}") + state = "on: the names below go to Caddy" if on else "off: every name goes to nginx (dev2-edge route off)" + return HEADER + f"""# The edge ({state}). +# nginx's stream module owns :443. It reads the TLS server name (ssl_preread; the TLS +# itself is not terminated here) and passes the connection on, with a PROXY protocol +# header carrying the visitor's address, to Caddy or to nginx's own https servers. +stream {{ + map $ssl_preread_server_name $dev2_edge_backend {{ + hostnames; + default {nginx}; +""" + "\n".join(rows) + ("\n" if rows else "") + f""" }} + + server {{ + listen 443; + listen [::]:443; + ssl_preread on; + preread_timeout 10s; + proxy_pass $dev2_edge_backend; + proxy_protocol on; + proxy_connect_timeout 5s; + # Never stricter than the longest proxy_read_timeout behind it (86400s). + proxy_timeout {s.get('stream_timeout', '24h')}; + }} +}} +""" + + +def render_realip(s): + return HEADER + f"""# Every https server listens on 127.0.0.1:{s['ports']['nginx']} behind the edge stream proxy, +# which sends the visitor's address in a PROXY protocol header. Without these two lines +# $remote_addr is 127.0.0.1 for every https visitor: rate limits, logs and ThreatCrush +# would all see one client. +set_real_ip_from 127.0.0.1; +real_ip_header proxy_protocol; +# nginx-made redirects (a directory without its slash) would otherwise name :{s['ports']['nginx']}. +port_in_redirect off; +""" + + +def render_cert_sources(s): + out = [HEADER.rstrip("\n"), "# [ ]: read by caddy-edge-certs"] + for c in s["cert_sources"]: + out.append(" ".join([c["dir"]] + ([c["crt"], c["key"], c["as"]] if c.get("as") else []))) + return "\n".join(out) + "\n" + + +def caddy_site(site, s): + tls = "edge_tls_moshpit" if site.get("moshpit") else "edge_tls" + lines = [", ".join(f"https://{h}" for h in site["hosts"]) + " {", f"\timport {tls}"] + if site.get("max_body"): + lines += ["\trequest_body {", f"\t\tmax_size {site['max_body']}", "\t}"] + if site.get("gzip"): + lines.append("\tencode gzip") + if site.get("access_log"): + lines += ["\tlog {", f"\t\toutput file /var/log/caddy/{site['hosts'][0]}.access.log", "\t}"] + if "proxy" in site: + lines.append(f"\timport edge_proxy {int(site['proxy'])}") + elif "root" in site: + lines += [f"\troot * {site['root']}", "\tfile_server"] + else: + lines.append(f"\tredir {site['redirect']} 301") + return "\n".join(lines) + "\n}\n" + + +def render_caddyfile(s): + p = s["ports"] + return HEADER + f"""# Caddy only ever sees connections the nginx stream proxy hands it (127.0.0.1:{p['caddy']}, +# PROXY protocol first). It issues nothing: certificates are bundles caddy-edge-certs +# copies from certbot / pit-proxy-cert / moshpit-renew. Upstreams are the ones nginx used. +{{ +\tadmin localhost:{p['caddy_admin']} +\tauto_https off +\tpersist_config off +\tdefault_bind 127.0.0.1 +\thttp_port {p['caddy_http']} +\thttps_port {p['caddy']} +\tservers {{ +\t\tlistener_wrappers {{ +\t\t\tproxy_protocol {{ +\t\t\t\ttimeout 5s +\t\t\t\tallow 127.0.0.1/32 +\t\t\t}} +\t\t\ttls +\t\t}} +\t\t# No HTTP/3: UDP 443 is not routed here, and an Alt-Svc for :{p['caddy']} would point browsers nowhere. +\t\tprotocols h1 h2 +\t}} +}} + +(edge_tls) {{ +\ttls {{ +\t\tload /etc/caddy/edge-certs +\t}} +}} + +# Moshpit names were TLS 1.3 only on nginx (with X25519MLKEM768 first, Go's default too). +(edge_tls_moshpit) {{ +\ttls {{ +\t\tload /etc/caddy/edge-certs +\t\tprotocols tls1.3 +\t}} +}} + +(edge_proxy) {{ +\treverse_proxy 127.0.0.1:{{args[0]}} {{ +\t\theader_up X-Real-IP {{remote_host}} +\t\t# A reload (every certificate renewal) would otherwise cut websockets (pit-bbs) at once. +\t\tstream_close_delay 1h +\t}} +}} + +""" + "\n".join(caddy_site(site, s) for site in s["sites"]) + + +UNITS = ("caddy.service", "caddy-edge-certs.service", "caddy-edge-certs.path", "caddy-edge-certs.timer", + "nginx-edge-listen.service", "nginx-edge-listen.path") +SCRIPTS = ("nginx-edge-listen", "caddy-edge-certs", "edge-sweep", "edge-box") + + +def render(out, s, on=True): + """Every file the box needs, laid out as the box paths under out/root/.""" + files = { + "etc/nginx/stream-enabled/dev2-edge.conf": render_stream(s, on), + "etc/nginx/conf.d/00-edge-realip.conf": render_realip(s), + "etc/caddy/Caddyfile": render_caddyfile(s), + "etc/caddy/edge-cert-sources": render_cert_sources(s), + } + for u in UNITS: + with open(os.path.join(EDGE, "systemd", u)) as f: + files[f"etc/systemd/system/{u}"] = f.read() + with open(os.path.join(EDGE, "systemd", "nginx.service.d-dev2-edge.conf")) as f: + files["etc/systemd/system/nginx.service.d/dev2-edge.conf"] = f.read() + for name, body in files.items(): + p = os.path.join(out, "root", name) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as f: + f.write(body) + for sc in SCRIPTS: + d = os.path.join(out, "root", "usr/local/sbin", sc) + os.makedirs(os.path.dirname(d), exist_ok=True) + shutil.copy(os.path.join(EDGE, sc), d) + os.chmod(d, 0o755) + with open(os.path.join(out, "edge.json"), "w") as f: + json.dump(s, f, indent=1) + return sorted(files) + [f"usr/local/sbin/{x}" for x in SCRIPTS] + + +# ------------------------------------------------------------------ caddy binary + +def caddy_binary(version): + """The official static build, sha512-checked against the release's checksums file.""" + base = f"https://github.com/caddyserver/caddy/releases/download/v{version}" + tgz = f"caddy_{version}_linux_amd64.tar.gz" + d = os.path.join(CACHE, version) + binp = os.path.join(d, "caddy") + if os.path.exists(binp): + return binp + os.makedirs(d, exist_ok=True) + log(f"downloading Caddy {version}") + sums = urllib.request.urlopen(f"{base}/caddy_{version}_checksums.txt", timeout=60).read().decode() + want = next((l.split()[0] for l in sums.splitlines() if l.endswith(" " + tgz)), None) + if not want: + die(f"{tgz} is not in the checksums file") + data = urllib.request.urlopen(f"{base}/{tgz}", timeout=300).read() + if hashlib.sha512(data).hexdigest() != want: + die(f"{tgz}: sha512 mismatch") + with tempfile.TemporaryDirectory(dir=d) as t: + p = os.path.join(t, tgz) + with open(p, "wb") as f: + f.write(data) + with tarfile.open(p) as tf: + m = tf.getmember("caddy") + tf.extract(m, t, filter="data") if hasattr(tarfile, "data_filter") else tf.extract(m, t) + shutil.move(os.path.join(t, "caddy"), binp) + os.chmod(binp, 0o755) + return binp + + +# ------------------------------------------------------------------ remote + +def ssh_root(cmd, check=True, capture=False): + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", cmd], text=True, capture_output=capture) + if check and r.returncode != 0: + die(f"on {DEV2_HOST}: {cmd!r} exited {r.returncode}" + (f"\n{r.stderr}" if capture else "")) + return r + + +def ship(on=True, with_caddy=True): + """Render + copy the bundle to root@dev2:/root/dev2-edge//, return that path.""" + s = spec() + ts = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + with tempfile.TemporaryDirectory(prefix="dev2-edge-") as out: + render(out, s, on) + if with_caddy: + shutil.copy(caddy_binary(s["caddy_version"]), os.path.join(out, "caddy")) + remote = f"/root/dev2-edge/{ts}" + ssh_root(f"install -d -m 0700 {remote}") + tar = subprocess.Popen(["tar", "-C", out, "-czf", "-", "."], stdout=subprocess.PIPE) + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", f"tar -C {remote} -xzf -"], stdin=tar.stdout) + tar.wait() + if r.returncode or tar.returncode: + die("could not copy the bundle to dev2") + return remote + + +def cmd_render(a): + s = spec() + out = a.out or tempfile.mkdtemp(prefix="dev2-edge-render-") + for f in render(out, s, not a.off): + print(os.path.join(out, "root", f)) + + +def cmd_apply(a): + if not a.yes: + die("this changes production nginx on dev2: re-run with --yes (see `dev2-edge render` for what it writes)") + remote = ship() + log(f"bundle on dev2 at {remote}; running edge-box apply") + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", f"{remote}/root/usr/local/sbin/edge-box apply {remote}"]) + sys.exit(r.returncode) + + +def cmd_route(a): + remote = ship(on=(a.state == "on"), with_caddy=False) + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", f"{remote}/root/usr/local/sbin/edge-box route {remote}"]) + sys.exit(r.returncode) + + +def cmd_rollback(a): + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", "/usr/local/sbin/edge-box rollback " + shlex.quote(a.tarball or "")]) + sys.exit(r.returncode) + + +def cmd_sweep(a): + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", "edge-sweep --names > /root/dev2-edge/names.txt && edge-sweep /root/dev2-edge/names.txt"]) + sys.exit(r.returncode) + + +def cmd_status(a): + r = subprocess.run(SSH + [f"root@{DEV2_HOST}", "/usr/local/sbin/edge-box status"]) + sys.exit(r.returncode) + + +def main(): + ap = argparse.ArgumentParser(prog="dev2-edge", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("render"); p.add_argument("--out"); p.add_argument("--off", action="store_true"); p.set_defaults(fn=cmd_render) + p = sub.add_parser("apply"); p.add_argument("--yes", action="store_true"); p.set_defaults(fn=cmd_apply) + p = sub.add_parser("route"); p.add_argument("state", choices=["on", "off"]); p.set_defaults(fn=cmd_route) + p = sub.add_parser("rollback"); p.add_argument("tarball", nargs="?"); p.set_defaults(fn=cmd_rollback) + sub.add_parser("sweep").set_defaults(fn=cmd_sweep) + sub.add_parser("status").set_defaults(fn=cmd_status) + a = ap.parse_args() + a.fn(a) + + +if __name__ == "__main__": + main() diff --git a/dev2/dev2-site b/dev2/dev2-site index 37d9da4..5d8c58f 100755 --- a/dev2/dev2-site +++ b/dev2/dev2-site @@ -1568,6 +1568,19 @@ def keepalive_upstreams(conf, site_name): head, sep, rest = conf.partition("\nserver {") return head + "\n" + ups.rstrip("\n") + "\n" + sep + rest +def edge_listen(conf): + """With dev2-edge on, nginx's https servers listen behind the :443 stream proxy + (127.0.0.1:8444, PROXY protocol): render that form instead of `listen 443`. The + rewrite is dev2/edge/nginx-edge-listen's, the same one the box applies to vhosts + other tools write, so a re-render never fights it.""" + if ssh("root", "test -e /etc/nginx/stream-enabled/dev2-edge.conf && echo on", check=False).strip() != "on": + return conf + import importlib.machinery, importlib.util + loader = importlib.machinery.SourceFileLoader("nginx_edge_listen", os.path.join(HERE, "edge", "nginx-edge-listen")) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader(loader.name, loader)) + loader.exec_module(mod) + return mod.normalize(conf) + def vhost_sites(): """Registry sites (and sites.d-only entries) whose vhost is enabled on dev2.""" have = set(ssh("root", "ls -1 /etc/nginx/sites-enabled", check=False).split()) @@ -1589,6 +1602,7 @@ def cmd_vhost(args): for name in names: s = site(name) conf, domains = render_vhost(s) + conf = edge_listen(conf) f = f"/etc/nginx/sites-available/{s['site']}" if getattr(args, "dry_run", False): live = ssh("root", f"cat {shlex.quote(f)} 2>/dev/null || true", check=False) diff --git a/dev2/edge/caddy-edge-certs b/dev2/edge/caddy-edge-certs new file mode 100755 index 0000000..25a9205 --- /dev/null +++ b/dev2/edge/caddy-edge-certs @@ -0,0 +1,77 @@ +#!/bin/sh +# caddy-edge-certs: hand Caddy the certificates dev2 already keeps current. +# +# Installed as /usr/local/sbin/caddy-edge-certs by cli-tools/dev2/dev2-edge; run by +# caddy-edge-certs.path whenever a source directory changes, and daily by its timer. +# +# Caddy does not issue anything on dev2 (auto_https off). The certificates come from +# the renewals that existed before Caddy did, unchanged: +# pit.world + www certbot (nginx authenticator, port 80 stays on nginx) +# .pit.world (+bbs/social) pit-proxy-cert -> /etc/nginx/pit-proxy-certs +# Moshpit names moshpit-renew(.timer) -> /etc/ssl/moshpit[-riotcoder] +# Those keys are root-only, so this copies each certificate + key into one bundle +# (.pem, root:caddy 0640) under $OUT, which the Caddyfile loads with +# `tls { load ... }`, and reloads Caddy only when a bundle changed. +# +# Sources come from $SOURCES, one per line: [ ] +# Without the three extra fields every .crt with a matching .key in is used. +set -eu + +OUT="${EDGE_CERT_OUT:-/etc/caddy/edge-certs}" +SOURCES="${EDGE_CERT_SOURCES:-/etc/caddy/edge-cert-sources}" +GROUP="${EDGE_CERT_GROUP:-caddy}" +RELOAD="${EDGE_CERT_RELOAD:-systemctl reload caddy}" +OPENSSL="${OPENSSL:-openssl}" + +[ -f "$SOURCES" ] || { echo "caddy-edge-certs: no $SOURCES" >&2; exit 1; } +umask 027 +parent=$(dirname "$OUT") +mkdir -p "$parent" +tmp=$(mktemp -d "$parent/.edge-certs.XXXXXX") +trap 'rm -rf "$tmp"' EXIT + +bad=0 +add() { # + [ -s "$2" ] && [ -s "$3" ] || return 0 + # A pair that does not match would make Caddy refuse the whole reload: skip it, loudly. + c=$($OPENSSL x509 -in "$2" -noout -pubkey 2>/dev/null | $OPENSSL sha256 2>/dev/null) || c=x + k=$($OPENSSL pkey -in "$3" -pubout 2>/dev/null | $OPENSSL sha256 2>/dev/null) || k=y + if [ "$c" != "$k" ]; then + echo "caddy-edge-certs: $2 and $3 do not match, skipped" >&2 + bad=$((bad + 1)) + return 0 + fi + { cat "$2"; echo; cat "$3"; } > "$tmp/$1.pem" +} + +while read -r dir crt key name; do + case "$dir" in ''|'#'*) continue ;; esac + if [ -n "${name:-}" ]; then + add "$name" "$dir/$crt" "$dir/$key" + continue + fi + for c in "$dir"/*.crt; do + [ -f "$c" ] || continue + add "$(basename "$c" .crt)" "$c" "${c%.crt}.key" + done +done < "$SOURCES" + +chgrp -R "$GROUP" "$tmp" 2>/dev/null || true +chmod 0750 "$tmp" +find "$tmp" -type f -exec chmod 0640 {} + + +if [ -d "$OUT" ] && diff -rq "$OUT" "$tmp" >/dev/null 2>&1; then + exit 0 +fi +n=$(find "$tmp" -type f | wc -l | tr -d ' ') +if [ -d "$OUT" ]; then mv "$OUT" "$OUT.old.$$"; fi +mv "$tmp" "$OUT" +rm -rf "$OUT.old.$$" +trap - EXIT +if [ "$bad" = 0 ]; then echo "caddy-edge-certs: $n bundles in $OUT"; else echo "caddy-edge-certs: $n bundles in $OUT, $bad skipped"; fi +if [ -n "$RELOAD" ] && [ "${EDGE_CERT_NO_RELOAD:-0}" != 1 ]; then + # Not running yet (first install) is fine; anything else is a real failure. + if systemctl is-active --quiet caddy 2>/dev/null || [ "$RELOAD" != "systemctl reload caddy" ]; then + $RELOAD + fi +fi diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box new file mode 100755 index 0000000..acbf2da --- /dev/null +++ b/dev2/edge/edge-box @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +edge-box: the dev2 side of cli-tools/dev2/dev2-edge. Runs as root on dev2. + + edge-box apply BUNDLE backup, baseline sweep, Caddy up and checked on its own port, + nginx switched, sweep again; any status change rolls back + edge-box route BUNDLE install only BUNDLE's stream map (dev2-edge route on|off) + edge-box rollback [TARBALL] undo apply from its backup (default: the last one) + edge-box status + +BUNDLE is what `dev2-edge` rendered and copied: BUNDLE/root/, BUNDLE/caddy, +BUNDLE/edge.json. Every step prints what it does; nothing is left half-applied: a +failure after the backup restores it. +""" +import datetime as dt +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time + +STATE = "/root/dev2-edge" +BACKUPS = os.environ.get("EDGE_BACKUPS", "/home/anthony/backups") +NGINX_CONF = "/etc/nginx/nginx.conf" +STREAM_DIR = "/etc/nginx/stream-enabled" +STREAM_CONF = f"{STREAM_DIR}/dev2-edge.conf" +REALIP_CONF = "/etc/nginx/conf.d/00-edge-realip.conf" +CF_CONF = "/etc/nginx/conf.d/cloudflare-realip.conf" +DROPIN = "/etc/systemd/system/nginx.service.d/dev2-edge.conf" +UNITS = ["caddy.service", "caddy-edge-certs.service", "caddy-edge-certs.path", "caddy-edge-certs.timer", + "nginx-edge-listen.service", "nginx-edge-listen.path"] +WATCHERS = ["nginx-edge-listen.path", "caddy-edge-certs.path", "caddy-edge-certs.timer"] +STREAM_INCLUDE = "include /etc/nginx/stream-enabled/*.conf;" +CF_MARK = "# edge: real_ip_header is proxy_protocol now (00-edge-realip.conf); was: " + + +def say(msg): + print(f"edge-box: {msg}", flush=True) + + +def run(cmd, check=True, capture=False, **kw): + r = subprocess.run(cmd, shell=isinstance(cmd, str), text=True, capture_output=capture, **kw) + if check and r.returncode != 0: + raise RuntimeError(f"{cmd!r} exited {r.returncode}" + (f": {(r.stderr or r.stdout).strip()[-800:]}" if capture else "")) + return r + + +def put(src, dst, mode=0o644): + os.makedirs(os.path.dirname(dst), exist_ok=True) + tmp = dst + ".edge-tmp" + shutil.copyfile(src, tmp) + os.chmod(tmp, mode) + os.replace(tmp, dst) + + +def listening(port): + out = run(["ss", "-Hltn", f"sport = :{port}"], capture=True).stdout + return bool(out.strip()) + + +# ------------------------------------------------------------------ sweep + +def names(bundle): + env = dict(os.environ) + out = run(["sh", f"{bundle}/root/usr/local/sbin/edge-sweep", "--names"], capture=True).stdout.split() + # Caddy's exact hosts too (before Caddy exists the box Caddyfile is absent). + with open(f"{bundle}/root/etc/caddy/Caddyfile") as f: + for m in re.finditer(r"^https://([a-z0-9.-]+)", f.read(), re.M): + out.append(m.group(1)) + return sorted(set(n for n in out if re.fullmatch(r"[a-z0-9][a-z0-9.-]*\.[a-z0-9-]+", n))) + + +def sweep(bundle, hosts, path): + lst = path + ".names" + with open(lst, "w") as f: + f.write("\n".join(hosts) + "\n") + out = run(["sh", f"{bundle}/root/usr/local/sbin/edge-sweep", lst], capture=True).stdout + with open(path, "w") as f: + f.write(out) + return parse(out) + + +def parse(text): + rows = {} + for l in text.splitlines(): + p = l.split("\t") + if len(p) == 4: + rows[p[0]] = tuple(p[1:]) + return rows + + +def probe_one(bundle, host): + r = run(["sh", f"{bundle}/root/usr/local/sbin/edge-sweep", "/dev/null"], check=False, capture=True, + env=dict(os.environ, EDGE_SWEEP_ONE=host)) + p = r.stdout.strip().split("\t") + return tuple(p[1:]) if len(p) == 4 else ("000", "000", "000") + + +def compare(bundle, base, after, retries=3): + """Names whose (https verified, https -k, http) changed, after re-probing each a few times.""" + bad = {} + for n, want in base.items(): + got = after.get(n) + tries = 0 + while got != want and tries < retries: + time.sleep(2) + got = probe_one(bundle, n) + tries += 1 + if got != want: + bad[n] = (want, got) + return bad + + +# ------------------------------------------------------------------ pieces + +def backup(ts): + os.makedirs(BACKUPS, exist_ok=True) + tgz = f"{BACKUPS}/nginx-{ts}.tgz" + paths = ["etc/nginx"] + [p.lstrip("/") for p in ["/etc/caddy", "/etc/systemd/system/nginx.service.d"] if os.path.exists(p)] + paths += [f"etc/systemd/system/{u}" for u in UNITS if os.path.exists(f"/etc/systemd/system/{u}")] + run(["tar", "-czf", tgz, "-C", "/"] + paths) + try: + shutil.chown(tgz, "anthony", "anthony") + except (LookupError, PermissionError): + pass + os.chmod(tgz, 0o600) + os.makedirs(STATE, exist_ok=True) + with open(f"{STATE}/last-backup", "w") as f: + f.write(tgz + "\n") + say(f"backup: {tgz}") + return tgz + + +def edit_nginx_conf(spec): + with open(NGINX_CONF) as f: + old = f.read() + new = old + if STREAM_INCLUDE not in new: + anchor = "include /etc/nginx/modules-enabled/*.conf;" + if anchor not in new: + raise RuntimeError(f"{NGINX_CONF}: no `{anchor}` line to put the stream include after") + new = new.replace(anchor, anchor + "\n# dev2-edge: the :443 SNI router\n" + STREAM_INCLUDE, 1) + wc, nofile = spec["nginx"]["worker_connections"], spec["nginx"]["worker_rlimit_nofile"] + # Each https visitor is now three connections in nginx (visitor, stream->backend, backend). + new = re.sub(r"(\n\s*worker_connections\s+)(\d+)(\s*;)", lambda m: m.group(1) + str(max(int(m.group(2)), wc)) + m.group(3), new, count=1) + if not re.search(r"^\s*worker_rlimit_nofile\s", new, re.M): + new = re.sub(r"^(worker_processes[^\n]*\n)", r"\1worker_rlimit_nofile %d;\n" % nofile, new, count=1, flags=re.M) + if new != old: + tmp = NGINX_CONF + ".edge-tmp" + with open(tmp, "w") as f: + f.write(new) + os.chmod(tmp, 0o644) + os.replace(tmp, NGINX_CONF) + say("nginx.conf: stream include, worker_connections, worker_rlimit_nofile") + + +def edit_cloudflare(): + if not os.path.exists(CF_CONF): + return + with open(CF_CONF) as f: + old = f.read() + new = re.sub(r"^(\s*)(real_ip_header\s+CF-Connecting-IP\s*;)", lambda m: m.group(1) + CF_MARK + m.group(2), old, flags=re.M) + if new != old: + tmp = CF_CONF + ".edge-tmp" + with open(tmp, "w") as f: + f.write(new) + shutil.copymode(CF_CONF, tmp) + os.replace(tmp, CF_CONF) + say(f"{CF_CONF}: real_ip_header CF-Connecting-IP commented (one real_ip_header per level; no dev2 name is behind Cloudflare)") + + +def install_caddy(bundle, spec): + want = spec["caddy_version"] + have = run("/usr/local/bin/caddy version 2>/dev/null || true", capture=True).stdout + if f"v{want} " not in have + " ": + put(f"{bundle}/caddy", "/usr/local/bin/caddy", 0o755) + say(f"caddy {want} -> /usr/local/bin/caddy") + if run(["id", "caddy"], check=False, capture=True).returncode != 0: + run(["useradd", "--system", "--home-dir", "/var/lib/caddy", "--shell", "/usr/sbin/nologin", "--user-group", "caddy"]) + say("user caddy") + for d in ("/var/lib/caddy", "/var/log/caddy"): + os.makedirs(d, exist_ok=True) + shutil.chown(d, "caddy", "caddy") + os.chmod(d, 0o750) + os.makedirs("/etc/caddy", exist_ok=True) + + +def install_files(bundle, which): + root = f"{bundle}/root" + for rel in which: + src = f"{root}/{rel}" + mode = 0o755 if rel.startswith("usr/local/sbin/") else 0o644 + put(src, "/" + rel, mode) + + +def caddy_check(bundle, spec, base): + """Caddy alone, on its own port, before any visitor reaches it: same answers as nginx gave.""" + port = spec["ports"]["caddy"] + hosts = sorted(h for h in base if any(match(h, p) for site in spec["sites"] for p in site["hosts"]) + and h not in spec.get("keep_on_nginx", {})) + bad = {} + for h in hosts: + want = base[h][1] # https -k: certificate trust is checked by the after-sweep + got = "000" + for _ in range(3): + r = run(["curl", "-sk", "-o", "/dev/null", "-m", "20", "-w", "%{http_code}", "--haproxy-protocol", + "--connect-to", f"{h}:443:127.0.0.1:{port}", f"https://{h}/"], check=False, capture=True) + got = r.stdout.strip() or "000" + if got == want: + break + time.sleep(1) + if got != want: + bad[h] = (want, got) + return hosts, bad + + +def match(host, pattern): + if not pattern.startswith("*."): + return host == pattern + rx = "^" + re.escape(pattern).replace(r"\*", r"[a-z0-9-]+") + "$" + return re.match(rx, host) is not None + + +# ------------------------------------------------------------------ commands + +def apply(bundle): + if os.geteuid() != 0: + sys.exit("edge-box: run as root") + with open(f"{bundle}/edge.json") as f: + spec = json.load(f) + ts = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + work = f"{STATE}/{ts}" + os.makedirs(work, exist_ok=True) + edge_on = os.path.exists(STREAM_CONF) + + run("nginx -t -q", capture=True) + if not edge_on: + for p in (spec["ports"]["nginx"], spec["ports"]["caddy"], spec["ports"]["caddy_http"], spec["ports"]["caddy_admin"]): + if listening(p): + sys.exit(f"edge-box: port {p} is already in use; nothing changed") + + tgz = backup(ts) + try: + hosts = names(bundle) + say(f"baseline sweep of {len(hosts)} names (twice, to set flaky ones aside)") + b1 = sweep(bundle, hosts, f"{work}/before-1.tsv") + b2 = sweep(bundle, hosts, f"{work}/before-2.tsv") + flaky = sorted(n for n in hosts if b1.get(n) != b2.get(n)) + base = {n: v for n, v in b2.items() if n not in flaky} + say(f"baseline: {len(base)} steady, {len(flaky)} flaky (not judged): {' '.join(flaky)[:600]}") + + if run(["dpkg", "-s", "libnginx-mod-stream"], check=False, capture=True).returncode != 0: + say("apt-get install libnginx-mod-stream (nginx's own stream module, same version)") + run("DEBIAN_FRONTEND=noninteractive apt-get install -y -q libnginx-mod-stream", capture=True) + if not run("grep -ls ngx_stream_module /etc/nginx/modules-enabled/*.conf || true", capture=True).stdout.strip(): + raise RuntimeError("libnginx-mod-stream is installed but /etc/nginx/modules-enabled loads no ngx_stream_module") + run("nginx -t -q", capture=True) + + install_caddy(bundle, spec) + install_files(bundle, ["etc/caddy/Caddyfile", "etc/caddy/edge-cert-sources"] + + [f"usr/local/sbin/{x}" for x in ("nginx-edge-listen", "caddy-edge-certs", "edge-sweep", "edge-box")] + + [f"etc/systemd/system/{u}" for u in UNITS]) + run(["env", "EDGE_CERT_NO_RELOAD=1", "/usr/local/sbin/caddy-edge-certs"]) + # As the caddy user: proves it can read the bundles, and validate opens the access + # logs, which must not end up root-owned. + run(["runuser", "-u", "caddy", "--", "env", "HOME=/var/lib/caddy", "/usr/local/bin/caddy", "validate", + "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"], capture=True) + say("caddy validate (as caddy): ok") + run("systemctl daemon-reload") + run("systemctl enable caddy.service >/dev/null 2>&1; systemctl restart caddy.service") + time.sleep(2) + run("systemctl is-active --quiet caddy.service") + checked, bad = caddy_check(bundle, spec, base) + if bad: + raise RuntimeError("Caddy answers differently from nginx, before any traffic moved: " + json.dumps(bad)) + say(f"Caddy on 127.0.0.1:{spec['ports']['caddy']} answers like nginx for {len(checked)} names") + + # ---- the switch: one nginx reload hands :443 from the http servers to the stream. + install_files(bundle, ["etc/nginx/stream-enabled/dev2-edge.conf", "etc/nginx/conf.d/00-edge-realip.conf", + "etc/systemd/system/nginx.service.d/dev2-edge.conf"]) + edit_cloudflare() + edit_nginx_conf(spec) + out = run(["/usr/local/sbin/nginx-edge-listen"], capture=True).stdout + say(f"nginx-edge-listen: {len(out.splitlines())} files moved to 127.0.0.1:{spec['ports']['nginx']}") + run("nginx -t -q", capture=True) + run("systemctl daemon-reload") + elog = "/var/log/nginx/error.log" + mark = os.path.getsize(elog) if os.path.exists(elog) else 0 + run("systemctl reload nginx") + time.sleep(3) + owners = run(["ss", "-Hltnp", "sport = :443"], capture=True).stdout + if "nginx" not in owners or not listening(spec["ports"]["nginx"]): + raise RuntimeError("after the reload :443 or :8444 is not nginx's: " + owners) + # A reload that cannot bind keeps the old configuration and says so only here. + with open(elog, errors="replace") as f: + f.seek(mark) + new_err = f.read() + if "[emerg]" in new_err: + raise RuntimeError("nginx reload failed: " + new_err[-800:]) + + say("after sweep") + after = sweep(bundle, hosts, f"{work}/after.tsv") + bad = compare(bundle, base, after) + if bad: + raise RuntimeError(f"{len(bad)} names changed status: " + json.dumps(bad)[:3000]) + run("systemctl enable --now " + " ".join(WATCHERS) + " >/dev/null 2>&1") + say(f"done: {len(base)} names answer exactly as before; watchers on; backup {tgz}") + say(f"sweeps in {work}; rollback: /usr/local/sbin/edge-box rollback {tgz}") + except BaseException as e: + say(f"FAILED: {e}") + say("rolling back") + rollback(tgz) + sys.exit(1) + + +def reverse_listen(text): + """Undo nginx-edge-listen for a file written after the backup.""" + text = re.sub(r"^([ \t]*)# edge: \[::\]:443 belongs to the stream proxy \(nginx-edge-listen\)[ \t]*(.*)$", + lambda m: m.group(1) + "listen [::]:443 ssl;" + ((" # " + m.group(2)) if m.group(2) else ""), text, flags=re.M) + def back(m): + params = [p for p in m.group(1).split() if p != "proxy_protocol"] + return "listen 443 " + " ".join(params) + ";" + return re.sub(r"listen\s+127\.0\.0\.1:8444\s+([^;]*);", back, text) + + +def rollback(tgz=None): + if not tgz: + with open(f"{STATE}/last-backup") as f: + tgz = f.read().strip() + say(f"rollback from {tgz}") + run("systemctl disable --now " + " ".join(WATCHERS) + " >/dev/null 2>&1", check=False) + for p in (DROPIN,): + if os.path.exists(p): + os.unlink(p) + run("systemctl daemon-reload", check=False) + for p in (STREAM_CONF, REALIP_CONF): + if os.path.exists(p): + os.unlink(p) + # Restore what the backup had (certificates pit-proxy-cert copied since stay as they are). + with tarfile.open(tgz) as tf: + members = [m for m in tf.getmembers() if m.name.startswith("etc/nginx/") and not m.name.startswith("etc/nginx/pit-proxy-certs")] + tf.extractall("/", members=members, **({"filter": "fully_trusted"} if hasattr(tarfile, "data_filter") else {})) + had = {m.name for m in tf.getmembers()} + # A vhost written after the backup was moved behind the edge: move it back. + for p in glob.glob("/etc/nginx/sites-enabled/*") + glob.glob("/etc/nginx/conf.d/*.conf"): + r = os.path.realpath(p) + if not os.path.isfile(r): + continue + with open(r) as f: + t = f.read() + if "127.0.0.1:8444" in t: + n = reverse_listen(t) + with open(r + ".edge-tmp", "w") as f: + f.write(n) + shutil.copymode(r, r + ".edge-tmp") + os.replace(r + ".edge-tmp", r) + say(f"listen 443 restored in {r}") + if os.path.isdir(STREAM_DIR) and not os.listdir(STREAM_DIR) and "etc/nginx/stream-enabled" not in had: + os.rmdir(STREAM_DIR) + t = run("nginx -t", check=False, capture=True) + if t.returncode != 0: + say("nginx -t FAILED after restore, NOT reloading; nginx keeps running its current config:\n" + t.stderr) + sys.exit(2) + run("systemctl reload nginx") + time.sleep(3) + run("systemctl disable --now caddy.service >/dev/null 2>&1", check=False) + owners = run(["ss", "-Hltnp", "sport = :443"], capture=True).stdout + say("rolled back; :443 -> " + ("nginx" if "nginx" in owners else owners.strip())) + + +def route(bundle): + put(f"{bundle}/root/etc/nginx/stream-enabled/dev2-edge.conf", STREAM_CONF) + run("nginx -t -q", capture=True) + run("systemctl reload nginx") + on = sum(1 for l in open(STREAM_CONF) if ":8453;" in l) + say(f"stream map installed: {on} patterns to Caddy") + + +def status(): + print("stream map:", "present" if os.path.exists(STREAM_CONF) else "absent") + if os.path.exists(STREAM_CONF): + for l in open(STREAM_CONF): + if re.search(r"127\.0\.0\.1:\d+;", l) and "proxy_pass" not in l: + print(" " + l.strip()) + for u in ["caddy.service"] + WATCHERS: + print(f"{u}: {run(['systemctl', 'is-active', u], check=False, capture=True).stdout.strip()}") + print("bundles:", len(glob.glob("/etc/caddy/edge-certs/*.pem"))) + print(run("ss -Hltnp '( sport = :443 or sport = :8444 or sport = :8453 )' | awk '{print $4, $6}' | cut -c1-80", capture=True).stdout) + print("nginx-edge-listen --check:", run(["/usr/local/sbin/nginx-edge-listen", "--check"], check=False, capture=True).stdout.strip() or "clean") + + +def main(argv): + if not argv: + print(__doc__) + return 2 + cmd, rest = argv[0], argv[1:] + if cmd == "apply" and rest: + apply(rest[0]) + elif cmd == "route" and rest: + route(rest[0]) + elif cmd == "rollback": + rollback(rest[0] if rest and rest[0] else None) + elif cmd == "status": + status() + else: + print(__doc__) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/dev2/edge/edge-sweep b/dev2/edge/edge-sweep new file mode 100755 index 0000000..44d1f72 --- /dev/null +++ b/dev2/edge/edge-sweep @@ -0,0 +1,42 @@ +#!/bin/sh +# edge-sweep: the status code of https:/// and http:/// for every name. +# +# Installed as /usr/local/sbin/edge-sweep by cli-tools/dev2/dev2-edge, which runs it +# before and after an edge change and compares the two. Each name is pinned to +# $EDGE_IP (dev2's public address) so the request takes the same path a visitor's +# does (stream proxy -> Caddy or nginx), whatever public DNS says; Moshpit names are +# not in public DNS at all. +# +# edge-sweep NAMES_FILE > out.tsv name https (verified) https (-k) http +# edge-sweep --names every name nginx and Caddy serve (needs root) +set -eu +IP="${EDGE_IP:-23.95.228.174}" +JOBS="${EDGE_SWEEP_JOBS:-8}" +TIMEOUT="${EDGE_SWEEP_TIMEOUT:-20}" + +if [ "${1:-}" = "--names" ]; then + { + nginx -T 2>/dev/null | awk ' + /^[ \t]*server_name[ \t]/ { sub(/^[ \t]*server_name[ \t]+/, ""); sub(/;.*/, ""); n = split($0, a, /[ \t]+/); for (i = 1; i <= n; i++) print a[i] }' + # Hosts behind a regex server_name: one per certificate pit-proxy-cert made. + ls /etc/nginx/pit-proxy-certs 2>/dev/null | sed -n 's/\.crt$//p' + [ -f /etc/caddy/Caddyfile ] && sed -n 's/^https:\/\/\([a-z0-9.-]*\)\( {\|,\).*/\1/p' /etc/caddy/Caddyfile + } | grep -E '^[a-z0-9][a-z0-9.-]*\.[a-z0-9-]+$' | grep -v '^_$' | sort -u + exit 0 +fi + +[ -f "${1:-}" ] || { echo "usage: edge-sweep NAMES_FILE | --names" >&2; exit 2; } +probe() { + n="$1" + # Verified first; only when that fails (000: no answer, or a certificate this box + # does not trust) ask again without verification, so the two columns tell a TLS + # change apart from an app change. + v=$(curl -s -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:443:$IP" "https://$n/" 2>/dev/null) || true + k=$v + [ "${v:-000}" != 000 ] || k=$(curl -sk -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:443:$IP" "https://$n/" 2>/dev/null) || true + h=$(curl -s -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:80:$IP" "http://$n/" 2>/dev/null) || true + printf '%s\t%s\t%s\t%s\n' "$n" "${v:-000}" "${k:-000}" "${h:-000}" +} +# POSIX sh has no `export -f`: xargs re-enters this script once per name. +if [ "${EDGE_SWEEP_ONE:-}" ]; then probe "$EDGE_SWEEP_ONE"; exit 0; fi +grep -v '^[[:space:]]*$' "$1" | xargs -P "$JOBS" -I{} env EDGE_SWEEP_ONE={} sh "$0" "$1" | sort diff --git a/dev2/edge/edge.json b/dev2/edge/edge.json new file mode 100644 index 0000000..9763888 --- /dev/null +++ b/dev2/edge/edge.json @@ -0,0 +1,33 @@ +{ + "//": "dev2 edge: which TLS names the nginx stream proxy hands to Caddy, and what Caddy does with them. Rendered by dev2/dev2-edge. Every upstream is the one nginx used before the move (2026-10-19). Certificates are NOT issued by Caddy: caddy-edge-certs copies the files the existing renewals (certbot, pit-proxy-cert, moshpit-renew) already keep current, and reloads Caddy when they change.", + "ports": { "nginx": 8444, "caddy": 8453, "caddy_http": 8454, "caddy_admin": 2019 }, + "caddy_version": "2.11.7", + "stream_timeout": "24h", + "nginx": { "worker_connections": 4096, "worker_rlimit_nofile": 16384 }, + "cert_sources": [ + { "dir": "/etc/letsencrypt/live/pit.world", "crt": "fullchain.pem", "key": "privkey.pem", "as": "pit.world" }, + { "dir": "/etc/nginx/pit-proxy-certs" }, + { "dir": "/etc/ssl/moshpit" }, + { "dir": "/etc/ssl/moshpit-riotcoder" } + ], + "keep_on_nginx": { + "//": "Names under a routed pattern that stay on nginx for now, with why.", + "live.streaming.pit.world": "per-path routing to two nixamp servers with a 404 fallback (error_page @server2); move after it has its own Caddy rehearsal" + }, + "sites": [ + { "hosts": ["pit.world"], "proxy": 8787, "max_body": "1MiB" }, + { "hosts": ["www.pit.world"], "redirect": "https://pit.world{uri}" }, + { "hosts": ["ad.network.pit.world"], "proxy": 8800, "max_body": "1MiB", "gzip": true }, + { "hosts": ["*.*.pit.world", "*.*.*.pit.world"], "proxy": 8787, "max_body": "30MiB", "gzip": true, + "//": "pit-proxy: every .pit.world (+ bbs./social.) relayed by the pit.world app; one Let's Encrypt certificate per name from pit-proxy-cert" }, + { "hosts": ["built.indie"], "proxy": 8798, "max_body": "1MiB", "moshpit": true }, + { "hosts": ["family.ancestry", "family.tree"], "proxy": 8792, "max_body": "30MiB", "moshpit": true }, + { "hosts": ["pay.agent"], "proxy": 8796, "max_body": "64KiB", "moshpit": true }, + { "hosts": ["search.peptides"], "proxy": 8802, "max_body": "1MiB", "moshpit": true }, + { "hosts": ["seo.rank"], "proxy": 8790, "max_body": "64KiB", "moshpit": true }, + { "hosts": ["alt.2600"], "root": "/var/www/alt.2600", "moshpit": true, "access_log": true }, + { "hosts": ["auto.hacker"], "root": "/var/www/auto.hacker", "moshpit": true, "access_log": true }, + { "hosts": ["chovy.hacker"], "root": "/var/www/chovy.hacker", "moshpit": true, "access_log": true }, + { "hosts": ["profullstack.agent"], "root": "/var/www/profullstack.agent", "moshpit": true, "access_log": true } + ] +} diff --git a/dev2/edge/nginx-edge-listen b/dev2/edge/nginx-edge-listen new file mode 100755 index 0000000..4bfc26f --- /dev/null +++ b/dev2/edge/nginx-edge-listen @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +nginx-edge-listen: keep every nginx `listen 443` on dev2 behind the edge stream proxy. + +Installed as /usr/local/sbin/nginx-edge-listen by cli-tools/dev2/dev2-edge. + +With the edge on, nginx's `stream {}` block owns :443 and hands each connection, by +TLS server name, either to Caddy (pit.world, *.pit.world, Moshpit names) or back to +nginx's own http servers on 127.0.0.1:8444 with a PROXY protocol header. So an http +server block must say + + listen 127.0.0.1:8444 ssl proxy_protocol; + +and never `listen 443` (or `listen [::]:443`): `nginx -t` passes such a block, but the +reload that follows cannot bind :443 (the stream has it) and nginx keeps running the +OLD configuration, silently; a restart fails outright and takes every site down. + +Several tools still write `listen 443` (OpenInstall bin/install.sh, moshpit-proxy's +setup-origin.sh on every Moshpit renewal, certbot --nginx). This script rewrites those +lines in place, and runs before every nginx start and reload (systemd drop-in) and +whenever a vhost file changes (nginx-edge-listen.path), so they cannot stick. + + nginx-edge-listen rewrite, print each file changed + nginx-edge-listen --check change nothing; exit 1 if anything would change + nginx-edge-listen --reload rewrite, then `nginx -t && systemctl reload nginx` if anything changed + nginx-edge-listen --quiet no output + nginx-edge-listen FILE... only these files (tests, or one vhost) + +Rules, per `listen` directive: + listen 443 ...; / listen 0.0.0.0:443 ...; / listen :443 ...; / listen *:443 ...; + -> listen 127.0.0.1:8444 ; + listen [::]:443 ...; -> removed (commented), the stream listens on [::]:443 + listen ... quic ...; -> untouched (UDP, not the stream's business) +Anything else, including ports other than 443, is untouched. Idempotent. +""" +import glob +import os +import re +import subprocess +import sys +import tempfile + +BACKEND = os.environ.get("EDGE_NGINX_BACKEND", "127.0.0.1:8444") +NGINX = os.environ.get("EDGE_NGINX_DIR", "/etc/nginx") +MARK = "# edge: [::]:443 belongs to the stream proxy (nginx-edge-listen)" +# Socket options that may be given only once per address:port; the backend listen +# is shared by every server block, so none of them can come along. +SOCKET_OPTS = re.compile(r"^(ipv6only|reuseport|backlog|deferred|fastopen|rcvbuf|sndbuf|accept_filter|so_keepalive|setfib|bind)(=|$)") + +LISTEN = re.compile(r"(?P(?:^|[;{}\s]))listen(?P\s+)(?P\S+?)(?P(?:\s+[^;#]*?)?)\s*;") +V4_443 = re.compile(r"^(?:(?:\d{1,3}\.){3}\d{1,3}:|\*:)?443$") +V6_443 = re.compile(r"^\[[0-9a-fA-F:]*\]:443$") + + +def _line(line): + """Rewrite the listen directives on one line (comments left alone).""" + code, hash_, comment = line.partition("#") + if not re.search(r"\blisten\b", code): + return line + out, pos, v6 = [], 0, False + for m in LISTEN.finditer(code): + addr, params = m.group("addr"), m.group("params").split() + if "quic" in params: + continue + if V6_443.match(addr): + out.append(code[pos:m.start()] + m.group("lead")) + pos, v6 = m.end(), True + continue + if not V4_443.match(addr): + continue + keep = [p for p in params if not SOCKET_OPTS.match(p)] + if "ssl" not in keep: + keep.insert(0, "ssl") + if "proxy_protocol" not in keep: + keep.append("proxy_protocol") + out.append(code[pos:m.start()] + m.group("lead") + "listen" + m.group("sp") + BACKEND + " " + " ".join(keep) + ";") + pos = m.end() + if pos == 0: + return line + new = "".join(out) + code[pos:] + if v6 and not new.strip(): + # The whole line was `listen [::]:443 ...;` (+ maybe a comment): keep a marker. + indent = re.match(r"\s*", line).group(0) + return indent + MARK + ((" " + comment.strip()) if hash_ else "") + ("\n" if line.endswith("\n") else "") + return new + hash_ + comment + + +def normalize(text): + """The text with every listen 443 moved behind the edge. Pure; used by dev2-site too.""" + return "".join(_line(l) for l in text.splitlines(keepends=True)) + + +def files(): + """Real files nginx loads from sites-enabled and conf.d (symlinks resolved once).""" + seen, out = set(), [] + for p in sorted(glob.glob(os.path.join(NGINX, "sites-enabled", "*")) + glob.glob(os.path.join(NGINX, "conf.d", "*.conf"))): + r = os.path.realpath(p) + if r in seen or not os.path.isfile(r): + continue + seen.add(r) + out.append(r) + return out + + +def rewrite(path, check): + with open(path, encoding="utf-8", errors="surrogateescape") as f: + old = f.read() + new = normalize(old) + if new == old: + return False + if check: + return True + st = os.stat(path) + # Temp file + rename in the same directory: never `cmd file > file` (it truncates first). + fd, tmp = tempfile.mkstemp(prefix=".edge-", dir=os.path.dirname(path)) + try: + with os.fdopen(fd, "w", encoding="utf-8", errors="surrogateescape") as f: + f.write(new) + os.chmod(tmp, st.st_mode & 0o7777) + try: + os.chown(tmp, st.st_uid, st.st_gid) + except PermissionError: + pass + os.replace(tmp, path) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + return True + + +def main(argv): + flags = {a for a in argv if a.startswith("--")} + paths = [a for a in argv if not a.startswith("--")] + unknown = flags - {"--check", "--reload", "--quiet"} + if unknown: + print(f"nginx-edge-listen: unknown option {sorted(unknown)[0]}", file=sys.stderr) + return 2 + check, quiet = "--check" in flags, "--quiet" in flags + changed = [] + for p in paths or files(): + try: + if rewrite(p, check): + changed.append(p) + except OSError as e: + print(f"nginx-edge-listen: {p}: {e}", file=sys.stderr) + for p in changed: + if not quiet: + print(("would rewrite " if check else "rewrote ") + p) + if check: + return 1 if changed else 0 + if changed and "--reload" in flags: + t = subprocess.run(["nginx", "-t", "-q"]) + if t.returncode != 0: + print("nginx-edge-listen: nginx -t failed after the rewrite; not reloading", file=sys.stderr) + return 1 + return subprocess.run(["systemctl", "reload", "nginx"]).returncode + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/dev2/edge/systemd/caddy-edge-certs.path b/dev2/edge/systemd/caddy-edge-certs.path new file mode 100644 index 0000000..b66623e --- /dev/null +++ b/dev2/edge/systemd/caddy-edge-certs.path @@ -0,0 +1,15 @@ +# managed by cli-tools/dev2/dev2-edge +# certbot (pit.world), pit-proxy-cert (.pit.world) and moshpit-renew (Moshpit +# names) write here; Caddy picks the new certificate up within a second or two. +[Unit] +Description=Watch dev2's certificate directories for Caddy + +[Path] +PathChanged=/etc/nginx/pit-proxy-certs +PathChanged=/etc/ssl/moshpit +PathChanged=/etc/ssl/moshpit-riotcoder +PathChanged=/etc/letsencrypt/live/pit.world +Unit=caddy-edge-certs.service + +[Install] +WantedBy=paths.target diff --git a/dev2/edge/systemd/caddy-edge-certs.service b/dev2/edge/systemd/caddy-edge-certs.service new file mode 100644 index 0000000..dc77501 --- /dev/null +++ b/dev2/edge/systemd/caddy-edge-certs.service @@ -0,0 +1,7 @@ +# managed by cli-tools/dev2/dev2-edge +[Unit] +Description=Copy dev2's certificates into Caddy's bundles and reload Caddy if one changed + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/caddy-edge-certs diff --git a/dev2/edge/systemd/caddy-edge-certs.timer b/dev2/edge/systemd/caddy-edge-certs.timer new file mode 100644 index 0000000..2773500 --- /dev/null +++ b/dev2/edge/systemd/caddy-edge-certs.timer @@ -0,0 +1,12 @@ +# managed by cli-tools/dev2/dev2-edge +# Belt and braces for the .path unit: a renewal it missed is picked up within a day. +[Unit] +Description=Daily resync of Caddy's certificate bundles + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/dev2/edge/systemd/caddy.service b/dev2/edge/systemd/caddy.service new file mode 100644 index 0000000..bbc3855 --- /dev/null +++ b/dev2/edge/systemd/caddy.service @@ -0,0 +1,24 @@ +# managed by cli-tools/dev2/dev2-edge +[Unit] +Description=Caddy for pit.world and the Moshpit names, behind the dev2 edge stream proxy +Documentation=https://github.com/profullstack/cli-tools/blob/master/dev2/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +User=caddy +Group=caddy +Environment=HOME=/var/lib/caddy XDG_DATA_HOME=/var/lib XDG_CONFIG_HOME=/var/lib/caddy/.config +ExecStart=/usr/local/bin/caddy run --config /etc/caddy/Caddyfile --adapter caddyfile +ExecReload=/usr/local/bin/caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile --force +TimeoutStopSec=5s +LimitNOFILE=1048576 +PrivateTmp=true +ProtectSystem=full +NoNewPrivileges=true +Restart=on-failure +RestartSec=2s + +[Install] +WantedBy=multi-user.target diff --git a/dev2/edge/systemd/nginx-edge-listen.path b/dev2/edge/systemd/nginx-edge-listen.path new file mode 100644 index 0000000..298c18a --- /dev/null +++ b/dev2/edge/systemd/nginx-edge-listen.path @@ -0,0 +1,14 @@ +# managed by cli-tools/dev2/dev2-edge +# setup-origin.sh (every Moshpit renewal) rewrites its vhost with `listen 443` and runs +# `nginx -s reload` itself, past the systemd drop-in; this catches that within a second. +[Unit] +Description=Watch nginx's vhost directories for a `listen 443` to move behind the edge + +[Path] +PathChanged=/etc/nginx/sites-available +PathChanged=/etc/nginx/sites-enabled +PathChanged=/etc/nginx/conf.d +Unit=nginx-edge-listen.service + +[Install] +WantedBy=paths.target diff --git a/dev2/edge/systemd/nginx-edge-listen.service b/dev2/edge/systemd/nginx-edge-listen.service new file mode 100644 index 0000000..da4e71d --- /dev/null +++ b/dev2/edge/systemd/nginx-edge-listen.service @@ -0,0 +1,7 @@ +# managed by cli-tools/dev2/dev2-edge +[Unit] +Description=Move any new nginx `listen 443` behind the edge stream proxy, and reload + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/nginx-edge-listen --reload diff --git a/dev2/edge/systemd/nginx.service.d-dev2-edge.conf b/dev2/edge/systemd/nginx.service.d-dev2-edge.conf new file mode 100644 index 0000000..e1842f7 --- /dev/null +++ b/dev2/edge/systemd/nginx.service.d-dev2-edge.conf @@ -0,0 +1,12 @@ +# managed by cli-tools/dev2/dev2-edge (installed as /etc/systemd/system/nginx.service.d/dev2-edge.conf) +# Every start and reload first moves any `listen 443` a tool wrote behind the edge +# stream proxy: with the stream on :443, such a line passes `nginx -t` but the reload +# cannot bind and nginx silently keeps its old configuration (a start fails outright). +# The leading "-" keeps a broken rewrite from blocking nginx itself. +[Service] +ExecStartPre= +ExecStartPre=-/usr/local/sbin/nginx-edge-listen --quiet +ExecStartPre=/usr/sbin/nginx -t -q -g 'daemon on; master_process on;' +ExecReload= +ExecReload=-/usr/local/sbin/nginx-edge-listen --quiet +ExecReload=/usr/sbin/nginx -g 'daemon on; master_process on;' -s reload diff --git a/test/dev2-edge.test.ts b/test/dev2-edge.test.ts new file mode 100644 index 0000000..4fc45b7 --- /dev/null +++ b/test/dev2-edge.test.ts @@ -0,0 +1,239 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, rmSync, statSync, symlinkSync, writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +/** + * dev2/dev2-edge: pit.world + Moshpit names on Caddy behind an nginx stream proxy on :443, + * every other dev2 site on nginx (127.0.0.1:8444, PROXY protocol). These tests cover the + * pieces that decide whether production keeps answering: the `listen 443` rewrite every + * nginx start/reload runs, the rendered stream map and Caddyfile, and the certificate + * bundling that keeps Caddy on the renewals dev2 already has. + */ +const EDGE = resolve(__dirname, '../dev2/edge'); +const LISTEN = join(EDGE, 'nginx-edge-listen'); +const CERTS = join(EDGE, 'caddy-edge-certs'); +const CLI = resolve(__dirname, '../dev2/dev2-edge'); +const SPEC = JSON.parse(readFileSync(join(EDGE, 'edge.json'), 'utf8')); + +let dir = ''; +afterEach(() => { if (dir) rmSync(dir, { recursive: true, force: true }); dir = ''; }); +const tmp = () => (dir = mkdtempSync(join(tmpdir(), 'dev2-edge-'))); +const py = (args: string[], env: Record = {}) => + spawnSync('python3', args, { encoding: 'utf8', env: { ...process.env, ...env } }); + +describe('nginx-edge-listen', () => { + const VHOST = [ + 'server {', + ' listen 80;', + ' listen [::]:80;', + ' return 301 https://$host$request_uri;', + '}', + 'server {', + ' listen 443 ssl;', + ' listen [::]:443 ssl;', + ' http2 on;', + ' server_name a.example;', + '}', + 'server {', + ' listen 443 ssl http2 default_server reuseport backlog=511; # managed by Certbot', + ' listen [::]:443 ssl ipv6only=on; # managed by Certbot', + ' listen 0.0.0.0:443;', + ' listen 443 quic reuseport;', + ' listen 8443 ssl;', + ' # listen 443 ssl;', + '}', + 'server { listen 443 ssl; listen [::]:443 ssl; server_name one.line; }', + '', + ].join('\n'); + + function write(): string { + const d = tmp(); + mkdirSync(join(d, 'sites-available')); + mkdirSync(join(d, 'sites-enabled')); + mkdirSync(join(d, 'conf.d')); + writeFileSync(join(d, 'sites-available', 'a'), VHOST); + chmodSync(join(d, 'sites-available', 'a'), 0o640); + symlinkSync(join(d, 'sites-available', 'a'), join(d, 'sites-enabled', 'a')); + return d; + } + + it('moves every TCP listen 443 behind the edge and leaves the rest alone', () => { + const d = write(); + const r = py([LISTEN], { EDGE_NGINX_DIR: d }); + expect(r.status).toBe(0); + const out = readFileSync(join(d, 'sites-available', 'a'), 'utf8'); + expect(out).not.toMatch(/^\s*listen\s+(\S+:)?443\s+ssl/m); + expect(out).not.toMatch(/listen \[::\]:443/); + expect(out).toContain(' listen 127.0.0.1:8444 ssl proxy_protocol;\n'); + // socket options cannot repeat on the shared backend listen; server options stay + expect(out).toContain(' listen 127.0.0.1:8444 ssl http2 default_server proxy_protocol; # managed by Certbot'); + expect(out).toContain(' listen 127.0.0.1:8444 ssl proxy_protocol;\n listen 443 quic reuseport;'); + expect(out).toContain('listen 8443 ssl;'); + expect(out).toContain('# listen 443 ssl;'); + expect(out).toContain('listen 80;\n listen [::]:80;'); + expect(out).toContain('server { listen 127.0.0.1:8444 ssl proxy_protocol; server_name one.line; }'); + expect(out).toContain('# edge: [::]:443 belongs to the stream proxy (nginx-edge-listen) managed by Certbot'); + }); + + it('rewrites the target of a sites-enabled symlink in place, keeping link and mode', () => { + const d = write(); + py([LISTEN], { EDGE_NGINX_DIR: d }); + const link = join(d, 'sites-enabled', 'a'); + expect(lstatSync(link).isSymbolicLink()).toBe(true); + expect(readlinkSync(link)).toBe(join(d, 'sites-available', 'a')); + expect(statSync(join(d, 'sites-available', 'a')).mode & 0o777).toBe(0o640); + expect(readdirSync(join(d, 'sites-available'))).toEqual(['a']); // no temp file left + }); + + it('is idempotent, and --check reports whether anything is left', () => { + const d = write(); + expect(py([LISTEN, '--check'], { EDGE_NGINX_DIR: d }).status).toBe(1); + py([LISTEN], { EDGE_NGINX_DIR: d }); + const once = readFileSync(join(d, 'sites-available', 'a'), 'utf8'); + const again = py([LISTEN], { EDGE_NGINX_DIR: d }); + expect(again.stdout).toBe(''); + expect(readFileSync(join(d, 'sites-available', 'a'), 'utf8')).toBe(once); + expect(py([LISTEN, '--check'], { EDGE_NGINX_DIR: d }).status).toBe(0); + }); + + it("edge-box rollback's reverse rewrite gives a plain listen 443 back", () => { + const d = write(); + py([LISTEN], { EDGE_NGINX_DIR: d }); + const moved = readFileSync(join(d, 'sites-available', 'a'), 'utf8'); + const code = [ + 'import importlib.machinery, importlib.util, sys', + `l = importlib.machinery.SourceFileLoader("eb", ${JSON.stringify(join(EDGE, 'edge-box'))})`, + 'm = importlib.util.module_from_spec(importlib.util.spec_from_loader(l.name, l)); l.exec_module(m)', + 'sys.stdout.write(m.reverse_listen(sys.stdin.read()))', + ].join('\n'); + const back = execFileSync('python3', ['-c', code], { input: moved, encoding: 'utf8' }); + expect(back).not.toContain('8444'); + expect(back).toContain(' listen 443 ssl;\n listen [::]:443 ssl;\n http2 on;'); + }); +}); + +describe('dev2-edge render', () => { + function render(off = false) { + const d = tmp(); + const r = py([CLI, 'render', '--out', d, ...(off ? ['--off'] : [])]); + expect(r.status, r.stderr).toBe(0); + const f = (p: string) => readFileSync(join(d, 'root', p), 'utf8'); + return { d, stream: f('etc/nginx/stream-enabled/dev2-edge.conf'), caddy: f('etc/caddy/Caddyfile'), realip: f('etc/nginx/conf.d/00-edge-realip.conf'), sources: f('etc/caddy/edge-cert-sources') }; + } + + it('routes the pit.world names to Caddy and everything else to nginx, with PROXY protocol', () => { + const { stream } = render(); + expect(stream).toMatch(/listen 443;\s+listen \[::\]:443;\s+ssl_preread on;/); + expect(stream).toContain('proxy_protocol on;'); + expect(stream).toMatch(/default\s+127\.0\.0\.1:8444;/); + expect(stream).toMatch(/\n\s+pit\.world\s+127\.0\.0\.1:8453;/); + expect(stream).toMatch(/\n\s+\*\.pit\.world\s+127\.0\.0\.1:8453;/); + expect(stream).toMatch(/\n\s+chovy\.hacker\s+127\.0\.0\.1:8453;/); + // an exact name under a routed wildcard can stay on nginx + expect(stream).toMatch(/\n\s+live\.streaming\.pit\.world\s+127\.0\.0\.1:8444;/); + expect(stream).toContain('hostnames;'); + }); + + it('--off routes every name to nginx (the fast way back)', () => { + const { stream } = render(true); + expect(stream).not.toContain(':8453'); + expect(stream).toMatch(/default\s+127\.0\.0\.1:8444;/); + }); + + it('gives Caddy a site for every routed host, the upstream nginx used, and no ACME', () => { + const { caddy } = render(); + expect(caddy).toContain('auto_https off'); + expect(caddy).toContain('default_bind 127.0.0.1'); + expect(caddy).toContain('https_port 8453'); + expect(caddy).toMatch(/listener_wrappers \{\s+proxy_protocol \{[^}]*allow 127\.0\.0\.1\/32\s+\}\s+tls\s+\}/); + expect(caddy).toContain('protocols h1 h2'); + expect(caddy).not.toMatch(/on_demand|acme_ca|\bemail\b/); + expect(caddy).toContain('header_up X-Real-IP {remote_host}'); + for (const site of SPEC.sites) { + const head = site.hosts.map((h: string) => `https://${h}`).join(', ') + ' {'; + expect(caddy).toContain(head); + const block = caddy.slice(caddy.indexOf(head), caddy.indexOf('\n}\n', caddy.indexOf(head))); + if (site.proxy) expect(block).toContain(`import edge_proxy ${site.proxy}`); + if (site.root) expect(block).toContain(`root * ${site.root}`); + if (site.redirect) expect(block).toContain(`redir ${site.redirect} 301`); + if (site.max_body) expect(block).toContain(`max_size ${site.max_body}`); + expect(block).toContain(site.moshpit ? 'import edge_tls_moshpit' : 'import edge_tls'); + } + expect(caddy).toMatch(/\(edge_tls_moshpit\) \{\s+tls \{\s+load \/etc\/caddy\/edge-certs\s+protocols tls1\.3/); + }); + + it('every name the stream sends to Caddy has a Caddy site', () => { + const { stream } = render(); + const routed = [...stream.matchAll(/^\s+(\S+)\s+127\.0\.0\.1:8453;/gm)].map((m) => m[1]); + const hosts: string[] = SPEC.sites.flatMap((s: { hosts: string[] }) => s.hosts); + for (const r of routed) { + const ok = r.startsWith('*.') ? hosts.some((h) => h.startsWith('*.') && h.endsWith(r.slice(1))) : hosts.includes(r); + expect(ok, r).toBe(true); + } + }); + + it('restores the visitor address in nginx and keeps redirects off :8444', () => { + const { realip, sources } = render(); + expect(realip).toContain('set_real_ip_from 127.0.0.1;'); + expect(realip).toContain('real_ip_header proxy_protocol;'); + expect(realip).toContain('port_in_redirect off;'); + expect(sources).toContain('/etc/letsencrypt/live/pit.world fullchain.pem privkey.pem pit.world'); + expect(sources).toContain('/etc/nginx/pit-proxy-certs'); + expect(sources).toContain('/etc/ssl/moshpit'); + }); + + it('the Caddyfile adapts, when a caddy binary is on PATH', () => { + const which = spawnSync('sh', ['-c', 'command -v caddy'], { encoding: 'utf8' }); + if (which.status !== 0) return; + const { d } = render(); + const r = spawnSync('caddy', ['adapt', '--config', join(d, 'root/etc/caddy/Caddyfile'), '--adapter', 'caddyfile'], { encoding: 'utf8' }); + expect(r.status, r.stderr).toBe(0); + }); +}); + +describe('caddy-edge-certs', () => { + function pair(d: string, name: string, cn = name) { + execFileSync('openssl', ['req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes', '-days', '2', + '-subj', `/CN=${cn}`, '-keyout', join(d, `${name}.key`), '-out', join(d, `${name}.crt`)], { stdio: 'ignore' }); + } + + function setup() { + const d = tmp(); + const src = join(d, 'src'), le = join(d, 'le'); + mkdirSync(src); mkdirSync(le); + pair(src, 'a.b.pit.world'); + pair(src, 'chovy.hacker'); + pair(src, 'bad'); pair(src, 'other'); + writeFileSync(join(src, 'bad.key'), readFileSync(join(src, 'other.key'))); // mismatched pair + writeFileSync(join(src, 'moshpit-root-ca.crt'), readFileSync(join(src, 'chovy.hacker.crt'))); // no key: not served + pair(le, 'fullchain'); // as fullchain.crt/key; listed explicitly below + const sources = join(d, 'sources'); + writeFileSync(sources, `# test\n${src}\n${le} fullchain.crt fullchain.key pit.world\n`); + const gid = execFileSync('id', ['-gn'], { encoding: 'utf8' }).trim(); + const env = { EDGE_CERT_OUT: join(d, 'out', 'edge-certs'), EDGE_CERT_SOURCES: sources, EDGE_CERT_GROUP: gid, EDGE_CERT_RELOAD: 'echo RELOADED' }; + return { d, env }; + } + + it('bundles every matching pair, skips a mismatched one, and reloads once', () => { + const { d, env } = setup(); + const r = spawnSync('sh', [CERTS], { encoding: 'utf8', env: { ...process.env, ...env } }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain('RELOADED'); + expect(r.stderr).toContain('do not match'); + const out = join(d, 'out', 'edge-certs'); + expect(readdirSync(out).sort()).toEqual(['a.b.pit.world.pem', 'chovy.hacker.pem', 'other.pem', 'pit.world.pem']); + const b = readFileSync(join(out, 'chovy.hacker.pem'), 'utf8'); + expect(b).toContain('BEGIN CERTIFICATE'); + expect(b).toContain('PRIVATE KEY'); + expect(statSync(join(out, 'chovy.hacker.pem')).mode & 0o777).toBe(0o640); + + const again = spawnSync('sh', [CERTS], { encoding: 'utf8', env: { ...process.env, ...env } }); + expect(again.status).toBe(0); + expect(again.stdout).not.toContain('RELOADED'); + expect(existsSync(join(d, 'out', 'edge-certs.old'))).toBe(false); + }); +}); From cc2a5e342008dbe851ded57211e1e810737ea368 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:38:23 +0000 Subject: [PATCH 2/6] edge-box: backup tarball stays root 0600 (it holds the private keys) Co-Authored-By: Claude Opus 5.5 --- dev2/edge/edge-box | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box index acbf2da..31cc2fb 100755 --- a/dev2/edge/edge-box +++ b/dev2/edge/edge-box @@ -123,11 +123,12 @@ def backup(ts): tgz = f"{BACKUPS}/nginx-{ts}.tgz" paths = ["etc/nginx"] + [p.lstrip("/") for p in ["/etc/caddy", "/etc/systemd/system/nginx.service.d"] if os.path.exists(p)] paths += [f"etc/systemd/system/{u}" for u in UNITS if os.path.exists(f"/etc/systemd/system/{u}")] - run(["tar", "-czf", tgz, "-C", "/"] + paths) + # root 0600: it holds every private key under /etc/nginx. + old_umask = os.umask(0o077) try: - shutil.chown(tgz, "anthony", "anthony") - except (LookupError, PermissionError): - pass + run(["tar", "-czf", tgz, "-C", "/"] + paths) + finally: + os.umask(old_umask) os.chmod(tgz, 0o600) os.makedirs(STATE, exist_ok=True) with open(f"{STATE}/last-backup", "w") as f: From be7bfcf20457668c9d6125dd58ea61dccaaf2241 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:40:45 +0000 Subject: [PATCH 3/6] edge-box: only restore a backup apply made, only run a shipped bundle; route waits for the reload Co-Authored-By: Claude Opus 5.5 --- dev2/edge/edge-box | 32 ++++++++++++++++++++++++++------ dev2/edge/edge-sweep | 4 +++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box index 31cc2fb..b972022 100755 --- a/dev2/edge/edge-box +++ b/dev2/edge/edge-box @@ -329,10 +329,28 @@ def reverse_listen(text): return re.sub(r"listen\s+127\.0\.0\.1:8444\s+([^;]*);", back, text) +def backup_path(arg=None): + """A backup apply made: nginx-.tgz directly under BACKUPS, nothing else.""" + if not arg: + with open(os.path.join(STATE, "last-backup")) as f: + arg = f.read().strip() + base = os.path.realpath(BACKUPS) + p = os.path.realpath(arg) + if os.path.dirname(p) != base or not re.fullmatch(r"nginx-\d{8}T\d{6}Z\.tgz", os.path.basename(p)) or not os.path.isfile(p): + raise SystemExit(f"edge-box: {arg} is not a backup under {base}") + return p + + +def bundle_path(arg): + """A bundle dev2-edge shipped: a directory with edge.json and root/.""" + p = os.path.realpath(arg) + if not (os.path.isfile(os.path.join(p, "edge.json")) and os.path.isdir(os.path.join(p, "root"))): + raise SystemExit(f"edge-box: {arg} is not a dev2-edge bundle") + return p + + def rollback(tgz=None): - if not tgz: - with open(f"{STATE}/last-backup") as f: - tgz = f.read().strip() + tgz = backup_path(tgz) say(f"rollback from {tgz}") run("systemctl disable --now " + " ".join(WATCHERS) + " >/dev/null 2>&1", check=False) for p in (DROPIN,): @@ -378,7 +396,9 @@ def route(bundle): put(f"{bundle}/root/etc/nginx/stream-enabled/dev2-edge.conf", STREAM_CONF) run("nginx -t -q", capture=True) run("systemctl reload nginx") - on = sum(1 for l in open(STREAM_CONF) if ":8453;" in l) + time.sleep(2) # the reload is asynchronous: let the new workers take over before reporting + with open(STREAM_CONF) as f: + on = sum(1 for l in f if ":8453;" in l) say(f"stream map installed: {on} patterns to Caddy") @@ -401,9 +421,9 @@ def main(argv): return 2 cmd, rest = argv[0], argv[1:] if cmd == "apply" and rest: - apply(rest[0]) + apply(bundle_path(rest[0])) elif cmd == "route" and rest: - route(rest[0]) + route(bundle_path(rest[0])) elif cmd == "rollback": rollback(rest[0] if rest and rest[0] else None) elif cmd == "status": diff --git a/dev2/edge/edge-sweep b/dev2/edge/edge-sweep index 44d1f72..fa7eebf 100755 --- a/dev2/edge/edge-sweep +++ b/dev2/edge/edge-sweep @@ -34,7 +34,9 @@ probe() { v=$(curl -s -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:443:$IP" "https://$n/" 2>/dev/null) || true k=$v [ "${v:-000}" != 000 ] || k=$(curl -sk -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:443:$IP" "https://$n/" 2>/dev/null) || true - h=$(curl -s -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:80:$IP" "http://$n/" 2>/dev/null) || true + # Port 80 on purpose: nginx's plain-http answer (ACME, the redirect to https) is part of what must not change. + plain=http + h=$(curl -s -o /dev/null -m "$TIMEOUT" -w '%{http_code}' --resolve "$n:80:$IP" "$plain://$n/" 2>/dev/null) || true printf '%s\t%s\t%s\t%s\n' "$n" "${v:-000}" "${k:-000}" "${h:-000}" } # POSIX sh has no `export -f`: xargs re-enters this script once per name. From 80f4cfed6a4676dce7717ecb7003a50fdd304fa6 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:41:59 +0000 Subject: [PATCH 4/6] edge-box: backups live in one fixed place Co-Authored-By: Claude Opus 5.5 --- dev2/edge/edge-box | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box index b972022..bc747e0 100755 --- a/dev2/edge/edge-box +++ b/dev2/edge/edge-box @@ -25,7 +25,7 @@ import tempfile import time STATE = "/root/dev2-edge" -BACKUPS = os.environ.get("EDGE_BACKUPS", "/home/anthony/backups") +BACKUPS = "/home/anthony/backups" NGINX_CONF = "/etc/nginx/nginx.conf" STREAM_DIR = "/etc/nginx/stream-enabled" STREAM_CONF = f"{STREAM_DIR}/dev2-edge.conf" From a1551bef95d33e2c528d6104f55983d353a75278 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:43:27 +0000 Subject: [PATCH 5/6] edge-box: the last-backup pointer is a fixed path Co-Authored-By: Claude Opus 5.5 --- dev2/edge/edge-box | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box index bc747e0..7c79164 100755 --- a/dev2/edge/edge-box +++ b/dev2/edge/edge-box @@ -25,6 +25,7 @@ import tempfile import time STATE = "/root/dev2-edge" +LAST_BACKUP = "/root/dev2-edge/last-backup" BACKUPS = "/home/anthony/backups" NGINX_CONF = "/etc/nginx/nginx.conf" STREAM_DIR = "/etc/nginx/stream-enabled" @@ -131,7 +132,7 @@ def backup(ts): os.umask(old_umask) os.chmod(tgz, 0o600) os.makedirs(STATE, exist_ok=True) - with open(f"{STATE}/last-backup", "w") as f: + with open(LAST_BACKUP, "w") as f: f.write(tgz + "\n") say(f"backup: {tgz}") return tgz @@ -332,7 +333,7 @@ def reverse_listen(text): def backup_path(arg=None): """A backup apply made: nginx-.tgz directly under BACKUPS, nothing else.""" if not arg: - with open(os.path.join(STATE, "last-backup")) as f: + with open(LAST_BACKUP) as f: arg = f.read().strip() base = os.path.realpath(BACKUPS) p = os.path.realpath(arg) From 0d099bae1b0432312d7ecab54f71ddbaabf5e2c4 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 20:47:06 +0000 Subject: [PATCH 6/6] edge-box: a re-apply reloads a running Caddy instead of restarting it Co-Authored-By: Claude Opus 5.5 --- dev2/edge/edge-box | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev2/edge/edge-box b/dev2/edge/edge-box index 7c79164..0ff0451 100755 --- a/dev2/edge/edge-box +++ b/dev2/edge/edge-box @@ -274,7 +274,9 @@ def apply(bundle): "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"], capture=True) say("caddy validate (as caddy): ok") run("systemctl daemon-reload") - run("systemctl enable caddy.service >/dev/null 2>&1; systemctl restart caddy.service") + # Already serving (a re-apply after an edge.json change): reload, never drop connections. + run("systemctl enable caddy.service >/dev/null 2>&1; " + "if systemctl is-active --quiet caddy.service; then systemctl reload caddy.service; else systemctl restart caddy.service; fi") time.sleep(2) run("systemctl is-active --quiet caddy.service") checked, bad = caddy_check(bundle, spec, base)