feat: add stellarlight and atlas exporters with schemas and diff flag

apps/api/src/hackproof/cli.py

@@ -22,7 +22,19 @@ def main() -> None:
         "--projects", action="store_true", help="only scan repos that are already projects"
     )
     commands.add_parser("reconcile", help="link chain activity to repos and build projects")
-    commands.add_parser("export", help="write one yaml file per project into data and its schema")
+    export_cmd = commands.add_parser(
+        "export", help="write one yaml file per project into data and its schema"
+    )
+    export_cmd.add_argument(
+        "--target",
+        choices=("stellarlight", "atlas", "all"),
+        help="also export rows for stellarlight or atlas",
+    )
+    export_cmd.add_argument(
+        "--diff",
+        action="store_true",
+        help="print what changed since the previous target export",
+    )
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
@@ -37,4 +49,4 @@ def main() -> None:
         case "reconcile":
             asyncio.run(run_reconcile())
         case "export":
-            asyncio.run(run_export())
+            asyncio.run(run_export(args.target, args.diff))

apps/api/src/hackproof/export/atlas.py

@@ -0,0 +1,68 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from hackproof.export.models import Evidence, HackathonExport, Project
+
+
+class AtlasRow(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    repo: str
+    project: str
+    program: str
+    person_or_org: str
+    language: str | None
+    chain_evidence: list[Evidence]
+
+
+def to_atlas_row(program: str, project: Project) -> AtlasRow:
+    return AtlasRow(
+        repo=project.repo_url,
+        project=project.name,
+        program=program,
+        person_or_org=project.owner.login,
+        language=project.language,
+        chain_evidence=project.chain.evidence,
+    )
+
+
+def atlas_rows(export: HackathonExport) -> list[AtlasRow]:
+    return [to_atlas_row(export.hackathon.slug, p) for p in export.projects]
+
+
+def atlas_schema() -> dict[str, Any]:
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "$id": "https://github.com/devodii/hackproof/schema/atlas.schema.json",
+        **AtlasRow.model_json_schema(),
+    }
+
+
+def write_atlas(
+    export: HackathonExport,
+    data_dir: Path,
+    schema_dir: Path | None = None,
+    diff: bool = False,
+) -> list[dict[str, Any]]:
+    data_dir.mkdir(parents=True, exist_ok=True)
+    rows = [r.model_dump(mode="json") for r in atlas_rows(export)]
+    out_path = data_dir / "atlas.json"
+
+    if diff and out_path.exists():
+        previous = {item["repo"]: item for item in json.loads(out_path.read_text())}
+        current = {item["repo"]: item for item in rows}
+        added = sorted(set(current) - set(previous))
+        removed = sorted(set(previous) - set(current))
+        changed = sorted(k for k in set(current) & set(previous) if current[k] != previous[k])
+        print(f"atlas diff: +{len(added)} -{len(removed)} ~{len(changed)}")
+
+    out_path.write_text(f"{json.dumps(rows, indent=2)}\n")
+    target_schema_dir = schema_dir or data_dir
+    target_schema_dir.mkdir(parents=True, exist_ok=True)
+    (target_schema_dir / "atlas.schema.json").write_text(
+        f"{json.dumps(atlas_schema(), indent=2)}\n"
+    )
+    return rows

apps/api/src/hackproof/export/jobs.py

@@ -2,20 +2,27 @@
 
 from hackproof.config import ROOT
 from hackproof.db import open_pool, pool
+from hackproof.export.atlas import write_atlas
 from hackproof.export.build import build_export
 from hackproof.export.files import write_project_files
+from hackproof.export.stellarlight import write_stellarlight
 from hackproof.hackathon import get_hackathon
 
 log = logging.getLogger("hackproof.export")
 
 DATA = ROOT / "data"
+WEB_SCHEMA_DIR = ROOT / "apps" / "web" / "public" / "schema"
 
 
-async def run_export() -> None:
+async def run_export(target: str | None = None, diff: bool = False) -> None:
     hackathon = get_hackathon()
     await open_pool()
     async with pool().acquire() as connection:
         export = await build_export(connection, hackathon)
 
     count = write_project_files(export, DATA)
+    if target in ("stellarlight", "all"):
+        write_stellarlight(export, DATA, WEB_SCHEMA_DIR, diff=diff)
+    if target in ("atlas", "all"):
+        write_atlas(export, DATA, WEB_SCHEMA_DIR, diff=diff)
     log.info("wrote %s project files to %s", count, DATA / str(export.hackathon.starts_at.year))

apps/api/src/hackproof/export/stellarlight.py

@@ -0,0 +1,77 @@
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from hackproof.export.models import Evidence, HackathonExport, Project
+
+
+class StellarLightRow(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    event: str
+    name: str
+    tagline: str | None
+    repo: str
+    repo_verified: bool
+    verification_evidence: list[Evidence]
+    builders: list[str]
+    official_status: Literal["first", "runner-up", "submitted"]
+
+
+def to_stellarlight_row(event: str, project: Project) -> StellarLightRow:
+    status: Literal["first", "runner-up", "submitted"] = (
+        project.award if project.award is not None else "submitted"
+    )
+    return StellarLightRow(
+        id=f"pkg:github/{project.repo_full_name}",
+        event=event,
+        name=project.name,
+        tagline=project.description,
+        repo=project.repo_url,
+        repo_verified=project.tier == "verified",
+        verification_evidence=project.chain.evidence,
+        builders=[c.login for c in project.contributors],
+        official_status=status,
+    )
+
+
+def stellarlight_rows(export: HackathonExport) -> list[StellarLightRow]:
+    return [to_stellarlight_row(export.hackathon.slug, p) for p in export.projects]
+
+
+def stellarlight_schema() -> dict[str, Any]:
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "$id": "https://github.com/devodii/hackproof/schema/stellarlight.schema.json",
+        **StellarLightRow.model_json_schema(),
+    }
+
+
+def write_stellarlight(
+    export: HackathonExport,
+    data_dir: Path,
+    schema_dir: Path | None = None,
+    diff: bool = False,
+) -> list[dict[str, Any]]:
+    data_dir.mkdir(parents=True, exist_ok=True)
+    rows = [r.model_dump(mode="json") for r in stellarlight_rows(export)]
+    out_path = data_dir / "stellarlight.json"
+
+    if diff and out_path.exists():
+        previous = {item["id"]: item for item in json.loads(out_path.read_text())}
+        current = {item["id"]: item for item in rows}
+        added = sorted(set(current) - set(previous))
+        removed = sorted(set(previous) - set(current))
+        changed = sorted(k for k in set(current) & set(previous) if current[k] != previous[k])
+        print(f"stellarlight diff: +{len(added)} -{len(removed)} ~{len(changed)}")
+
+    out_path.write_text(f"{json.dumps(rows, indent=2)}\n")
+    target_schema_dir = schema_dir or data_dir
+    target_schema_dir.mkdir(parents=True, exist_ok=True)
+    (target_schema_dir / "stellarlight.schema.json").write_text(
+        f"{json.dumps(stellarlight_schema(), indent=2)}\n"
+    )
+    return rows

apps/api/tests/test_exporters.py

@@ -0,0 +1,36 @@
+import json
+
+import jsonschema
+
+from hackproof.export.atlas import atlas_schema, write_atlas
+from hackproof.export.stellarlight import stellarlight_schema, write_stellarlight
+from tests.test_export import populated_export
+
+
+async def test_stellarlight_and_atlas_exporters_validate_every_row_and_diff(
+    scratch_db, tmp_path, capsys
+):
+    export = await populated_export(scratch_db)
+    schema_dir = tmp_path / "schema"
+
+    sl_rows = write_stellarlight(export, tmp_path, schema_dir)
+    at_rows = write_atlas(export, tmp_path, schema_dir)
+
+    assert len(sl_rows) == 1
+    assert len(at_rows) == 1
+
+    sl_schema = json.loads((schema_dir / "stellarlight.schema.json").read_text())
+    at_schema = json.loads((schema_dir / "atlas.schema.json").read_text())
+    assert sl_schema == stellarlight_schema()
+    assert at_schema == atlas_schema()
+
+    for row in sl_rows:
+        jsonschema.validate(row, sl_schema)
+    for row in at_rows:
+        jsonschema.validate(row, at_schema)
+
+    write_stellarlight(export, tmp_path, schema_dir, diff=True)
+    write_atlas(export, tmp_path, schema_dir, diff=True)
+    out = capsys.readouterr().out
+    assert "stellarlight diff: +0 -0 ~0" in out
+    assert "atlas diff: +0 -0 ~0" in out