diff --git a/apps/api/src/hackproof/cli.py b/apps/api/src/hackproof/cli.py index 2afb2dd..9a284f4 100644 --- a/apps/api/src/hackproof/cli.py +++ b/apps/api/src/hackproof/cli.py @@ -22,7 +22,19 @@ def main() -> None: "--projects", action="store_true", help="only scan repos that are already projects" ) commands.add_parser("reconcile", help="link chain activity to repos and build projects") - commands.add_parser("export", help="write one yaml file per project into data and its schema") + export_cmd = commands.add_parser( + "export", help="write one yaml file per project into data and its schema" + ) + export_cmd.add_argument( + "--target", + choices=("stellarlight", "atlas", "all"), + help="also export rows for stellarlight or atlas", + ) + export_cmd.add_argument( + "--diff", + action="store_true", + help="print what changed since the previous target export", + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") @@ -37,4 +49,4 @@ def main() -> None: case "reconcile": asyncio.run(run_reconcile()) case "export": - asyncio.run(run_export()) + asyncio.run(run_export(args.target, args.diff)) diff --git a/apps/api/src/hackproof/export/atlas.py b/apps/api/src/hackproof/export/atlas.py new file mode 100644 index 0000000..a195c3d --- /dev/null +++ b/apps/api/src/hackproof/export/atlas.py @@ -0,0 +1,68 @@ +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from hackproof.export.models import Evidence, HackathonExport, Project + + +class AtlasRow(BaseModel): + model_config = ConfigDict(extra="forbid") + + repo: str + project: str + program: str + person_or_org: str + language: str | None + chain_evidence: list[Evidence] + + +def to_atlas_row(program: str, project: Project) -> AtlasRow: + return AtlasRow( + repo=project.repo_url, + project=project.name, + program=program, + person_or_org=project.owner.login, + language=project.language, + chain_evidence=project.chain.evidence, + ) + + +def atlas_rows(export: HackathonExport) -> list[AtlasRow]: + return [to_atlas_row(export.hackathon.slug, p) for p in export.projects] + + +def atlas_schema() -> dict[str, Any]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/devodii/hackproof/schema/atlas.schema.json", + **AtlasRow.model_json_schema(), + } + + +def write_atlas( + export: HackathonExport, + data_dir: Path, + schema_dir: Path | None = None, + diff: bool = False, +) -> list[dict[str, Any]]: + data_dir.mkdir(parents=True, exist_ok=True) + rows = [r.model_dump(mode="json") for r in atlas_rows(export)] + out_path = data_dir / "atlas.json" + + if diff and out_path.exists(): + previous = {item["repo"]: item for item in json.loads(out_path.read_text())} + current = {item["repo"]: item for item in rows} + added = sorted(set(current) - set(previous)) + removed = sorted(set(previous) - set(current)) + changed = sorted(k for k in set(current) & set(previous) if current[k] != previous[k]) + print(f"atlas diff: +{len(added)} -{len(removed)} ~{len(changed)}") + + out_path.write_text(f"{json.dumps(rows, indent=2)}\n") + target_schema_dir = schema_dir or data_dir + target_schema_dir.mkdir(parents=True, exist_ok=True) + (target_schema_dir / "atlas.schema.json").write_text( + f"{json.dumps(atlas_schema(), indent=2)}\n" + ) + return rows diff --git a/apps/api/src/hackproof/export/jobs.py b/apps/api/src/hackproof/export/jobs.py index 7e543f6..647397d 100644 --- a/apps/api/src/hackproof/export/jobs.py +++ b/apps/api/src/hackproof/export/jobs.py @@ -2,20 +2,27 @@ from hackproof.config import ROOT from hackproof.db import open_pool, pool +from hackproof.export.atlas import write_atlas from hackproof.export.build import build_export from hackproof.export.files import write_project_files +from hackproof.export.stellarlight import write_stellarlight from hackproof.hackathon import get_hackathon log = logging.getLogger("hackproof.export") DATA = ROOT / "data" +WEB_SCHEMA_DIR = ROOT / "apps" / "web" / "public" / "schema" -async def run_export() -> None: +async def run_export(target: str | None = None, diff: bool = False) -> None: hackathon = get_hackathon() await open_pool() async with pool().acquire() as connection: export = await build_export(connection, hackathon) count = write_project_files(export, DATA) + if target in ("stellarlight", "all"): + write_stellarlight(export, DATA, WEB_SCHEMA_DIR, diff=diff) + if target in ("atlas", "all"): + write_atlas(export, DATA, WEB_SCHEMA_DIR, diff=diff) log.info("wrote %s project files to %s", count, DATA / str(export.hackathon.starts_at.year)) diff --git a/apps/api/src/hackproof/export/stellarlight.py b/apps/api/src/hackproof/export/stellarlight.py new file mode 100644 index 0000000..4465b0b --- /dev/null +++ b/apps/api/src/hackproof/export/stellarlight.py @@ -0,0 +1,77 @@ +import json +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict + +from hackproof.export.models import Evidence, HackathonExport, Project + + +class StellarLightRow(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + event: str + name: str + tagline: str | None + repo: str + repo_verified: bool + verification_evidence: list[Evidence] + builders: list[str] + official_status: Literal["first", "runner-up", "submitted"] + + +def to_stellarlight_row(event: str, project: Project) -> StellarLightRow: + status: Literal["first", "runner-up", "submitted"] = ( + project.award if project.award is not None else "submitted" + ) + return StellarLightRow( + id=f"pkg:github/{project.repo_full_name}", + event=event, + name=project.name, + tagline=project.description, + repo=project.repo_url, + repo_verified=project.tier == "verified", + verification_evidence=project.chain.evidence, + builders=[c.login for c in project.contributors], + official_status=status, + ) + + +def stellarlight_rows(export: HackathonExport) -> list[StellarLightRow]: + return [to_stellarlight_row(export.hackathon.slug, p) for p in export.projects] + + +def stellarlight_schema() -> dict[str, Any]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/devodii/hackproof/schema/stellarlight.schema.json", + **StellarLightRow.model_json_schema(), + } + + +def write_stellarlight( + export: HackathonExport, + data_dir: Path, + schema_dir: Path | None = None, + diff: bool = False, +) -> list[dict[str, Any]]: + data_dir.mkdir(parents=True, exist_ok=True) + rows = [r.model_dump(mode="json") for r in stellarlight_rows(export)] + out_path = data_dir / "stellarlight.json" + + if diff and out_path.exists(): + previous = {item["id"]: item for item in json.loads(out_path.read_text())} + current = {item["id"]: item for item in rows} + added = sorted(set(current) - set(previous)) + removed = sorted(set(previous) - set(current)) + changed = sorted(k for k in set(current) & set(previous) if current[k] != previous[k]) + print(f"stellarlight diff: +{len(added)} -{len(removed)} ~{len(changed)}") + + out_path.write_text(f"{json.dumps(rows, indent=2)}\n") + target_schema_dir = schema_dir or data_dir + target_schema_dir.mkdir(parents=True, exist_ok=True) + (target_schema_dir / "stellarlight.schema.json").write_text( + f"{json.dumps(stellarlight_schema(), indent=2)}\n" + ) + return rows diff --git a/apps/api/tests/test_exporters.py b/apps/api/tests/test_exporters.py new file mode 100644 index 0000000..24fa89d --- /dev/null +++ b/apps/api/tests/test_exporters.py @@ -0,0 +1,36 @@ +import json + +import jsonschema + +from hackproof.export.atlas import atlas_schema, write_atlas +from hackproof.export.stellarlight import stellarlight_schema, write_stellarlight +from tests.test_export import populated_export + + +async def test_stellarlight_and_atlas_exporters_validate_every_row_and_diff( + scratch_db, tmp_path, capsys +): + export = await populated_export(scratch_db) + schema_dir = tmp_path / "schema" + + sl_rows = write_stellarlight(export, tmp_path, schema_dir) + at_rows = write_atlas(export, tmp_path, schema_dir) + + assert len(sl_rows) == 1 + assert len(at_rows) == 1 + + sl_schema = json.loads((schema_dir / "stellarlight.schema.json").read_text()) + at_schema = json.loads((schema_dir / "atlas.schema.json").read_text()) + assert sl_schema == stellarlight_schema() + assert at_schema == atlas_schema() + + for row in sl_rows: + jsonschema.validate(row, sl_schema) + for row in at_rows: + jsonschema.validate(row, at_schema) + + write_stellarlight(export, tmp_path, schema_dir, diff=True) + write_atlas(export, tmp_path, schema_dir, diff=True) + out = capsys.readouterr().out + assert "stellarlight diff: +0 -0 ~0" in out + assert "atlas diff: +0 -0 ~0" in out