feat: add sample dataset and one-shot compose seed service

apps/api/src/hackproof/cli.py

@@ -6,6 +6,7 @@
 from hackproof.github.jobs import run_discover, run_scan
 from hackproof.indexer.sync import run_sync
 from hackproof.linking.reconcile import run_reconcile
+from hackproof.sample import run_dump_sample, run_seed
 
 
 def main() -> None:
@@ -23,6 +24,10 @@ def main() -> None:
     )
     commands.add_parser("reconcile", help="link chain activity to repos and build projects")
     commands.add_parser("export", help="write one yaml file per project into data and its schema")
+    commands.add_parser("seed", help="load sample data when the projects table is empty")
+    commands.add_parser(
+        "dump-sample", help="regenerate data/sample/sample.sql from the database"
+    )
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
@@ -38,3 +43,7 @@ def main() -> None:
             asyncio.run(run_reconcile())
         case "export":
             asyncio.run(run_export())
+        case "seed":
+            asyncio.run(run_seed())
+        case "dump-sample":
+            asyncio.run(run_dump_sample())

apps/api/src/hackproof/sample.py

@@ -0,0 +1,217 @@
+from datetime import datetime
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from hackproof.config import get_settings
+from hackproof.db import close_pool, open_pool, pool
+
+logger = logging.getLogger(__name__)
+
+PRIVATE_REPO_COLUMNS = {"owner_avatar", "owner_location", "country"}
+PRIVATE_CONTRIBUTOR_COLUMNS = {"avatar_url"}
+
+
+class SampleSummary(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    projects: int
+    contracts: int
+    loaded: bool
+
+
+def default_sample_path() -> Path:
+    return get_settings().data_dir / "sample" / "sample.sql"
+
+
+def sanitize_repo_row(row: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: (None if key in PRIVATE_REPO_COLUMNS else val)
+        for key, val in row.items()
+    }
+
+
+def sanitize_contributor_row(row: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: (None if key in PRIVATE_CONTRIBUTOR_COLUMNS else val)
+        for key, val in row.items()
+    }
+
+
+def _sql_literal(value: Any) -> str:
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, datetime):
+        return f"'{value.isoformat()}'"
+    if isinstance(value, list):
+        items = ", ".join(_sql_literal(str(item)) for item in value)
+        return f"ARRAY[{items}]::text[]"
+    if isinstance(value, dict):
+        encoded = json.dumps(value).replace("'", "''")
+        return f"'{encoded}'::jsonb"
+    escaped = str(value).replace("'", "''")
+    return f"'{escaped}'"
+
+
+def format_insert_block(
+    table: str,
+    columns: list[str],
+    rows: list[dict[str, Any]],
+    conflict_cols: list[str],
+) -> str:
+    if not rows:
+        return ""
+    col_list = ", ".join(columns)
+    values_lines = []
+    for row in rows:
+        vals = ", ".join(_sql_literal(row.get(col)) for col in columns)
+        values_lines.append(f"    ({vals})")
+    conflict = ", ".join(conflict_cols)
+    body = ",\n".join(values_lines)
+    return (
+        f"INSERT INTO {table} ({col_list}) VALUES\n"
+        f"{body}\n"
+        f"ON CONFLICT ({conflict}) DO NOTHING;\n"
+    )
+
+
+async def load_sample_if_empty(sql_path: Path | None = None) -> SampleSummary:
+    target = sql_path or default_sample_path()
+    async with pool().acquire() as conn:
+        existing = await conn.fetchval("SELECT count(*) FROM projects")
+        if existing and existing > 0:
+            contracts_count = await conn.fetchval("SELECT count(*) FROM contracts")
+            logger.info("projects table already has %d rows; skipping sample seed", existing)
+            return SampleSummary(
+                projects=int(existing),
+                contracts=int(contracts_count or 0),
+                loaded=False,
+            )
+        sql_text = target.read_text(encoding="utf-8")
+        async with conn.transaction():
+            await conn.execute(sql_text)
+        projects_count = await conn.fetchval("SELECT count(*) FROM projects")
+        contracts_count = await conn.fetchval("SELECT count(*) FROM contracts")
+        logger.info(
+            "loaded sample dataset (%d projects, %d contracts)",
+            projects_count,
+            contracts_count,
+        )
+        return SampleSummary(
+            projects=int(projects_count or 0),
+            contracts=int(contracts_count or 0),
+            loaded=True,
+        )
+
+
+async def dump_sample_from_db(
+    output_path: Path | None = None,
+    project_limit: int = 10,
+    contract_limit: int = 20,
+) -> Path:
+    target = output_path or default_sample_path()
+    async with pool().acquire() as conn:
+        project_rows = [
+            dict(r)
+            for r in await conn.fetch(
+                "SELECT * FROM projects ORDER BY contracts DESC, project_id LIMIT $1",
+                project_limit,
+            )
+        ]
+        full_names = [r["full_name"] for r in project_rows if r.get("full_name")]
+        repo_rows = [
+            sanitize_repo_row(dict(r))
+            for r in await conn.fetch(
+                "SELECT * FROM repos WHERE full_name = ANY($1::text[]) ORDER BY full_name",
+                full_names,
+            )
+        ]
+        contract_rows = [
+            dict(r)
+            for r in await conn.fetch(
+                "SELECT * FROM contracts ORDER BY deployed_at ASC LIMIT $1",
+                contract_limit,
+            )
+        ]
+        link_rows = [
+            dict(r)
+            for r in await conn.fetch(
+                "SELECT * FROM links WHERE full_name = ANY($1::text[])"
+                " ORDER BY full_name, subject_kind, subject",
+                full_names,
+            )
+        ]
+        contributor_rows = [
+            sanitize_contributor_row(dict(r))
+            for r in await conn.fetch(
+                "SELECT * FROM repo_contributors WHERE full_name = ANY($1::text[])"
+                " ORDER BY full_name, contributions DESC",
+                full_names,
+            )
+        ]
+
+    blocks = [
+        "-- HackProof anonymized sample dataset\n"
+        "-- Contains only public GitHub handles and public testnet contract metadata.\n",
+    ]
+    if repo_rows:
+        blocks.append(
+            format_insert_block("repos", list(repo_rows[0].keys()), repo_rows, ["full_name"])
+        )
+    if project_rows:
+        blocks.append(
+            format_insert_block(
+                "projects", list(project_rows[0].keys()), project_rows, ["project_id"]
+            )
+        )
+    if contract_rows:
+        blocks.append(
+            format_insert_block(
+                "contracts", list(contract_rows[0].keys()), contract_rows, ["contract_id"]
+            )
+        )
+    if link_rows:
+        blocks.append(
+            format_insert_block(
+                "links",
+                list(link_rows[0].keys()),
+                link_rows,
+                ["subject_kind", "subject", "full_name"],
+            )
+        )
+    if contributor_rows:
+        blocks.append(
+            format_insert_block(
+                "repo_contributors",
+                list(contributor_rows[0].keys()),
+                contributor_rows,
+                ["full_name", "login"],
+            )
+        )
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text("\n".join(b for b in blocks if b), encoding="utf-8")
+    return target
+
+
+async def run_seed(sql_path: Path | None = None) -> SampleSummary:
+    await open_pool()
+    try:
+        return await load_sample_if_empty(sql_path)
+    finally:
+        await close_pool()
+
+
+async def run_dump_sample(output_path: Path | None = None) -> Path:
+    await open_pool()
+    try:
+        return await dump_sample_from_db(output_path)
+    finally:
+        await close_pool()

apps/api/tests/test_sample.py

@@ -0,0 +1,52 @@
+from hackproof.sample import (
+    default_sample_path,
+    format_insert_block,
+    sanitize_contributor_row,
+    sanitize_repo_row,
+)
+
+
+def test_sample_sql_file_contains_ten_projects_and_twenty_contracts():
+    path = default_sample_path()
+    assert path.exists()
+    sql = path.read_text(encoding="utf-8")
+    assert "INSERT INTO projects" in sql
+    assert "INSERT INTO contracts" in sql
+    assert sql.count("'repo:") == 10
+    assert sql.count("'CSAMPLECONTRACT") >= 20
+    assert "avatar" not in sql.split("VALUES")[1].lower()
+
+
+def test_sanitize_strips_personal_profile_fields():
+    repo = {
+        "full_name": "ygk94/4ward",
+        "owner": "ygk94",
+        "owner_avatar": "https://avatars.githubusercontent.com/u/1",
+        "owner_location": "Lisbon",
+        "country": "PT",
+    }
+    cleaned_repo = sanitize_repo_row(repo)
+    assert cleaned_repo["owner"] == "ygk94"
+    assert cleaned_repo["owner_avatar"] is None
+    assert cleaned_repo["owner_location"] is None
+    assert cleaned_repo["country"] is None
+
+    contributor = {
+        "full_name": "ygk94/4ward",
+        "login": "ygk94",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1",
+        "contributions": 96,
+    }
+    cleaned_contributor = sanitize_contributor_row(contributor)
+    assert cleaned_contributor["login"] == "ygk94"
+    assert cleaned_contributor["avatar_url"] is None
+
+
+def test_format_insert_block_escapes_and_renders_conflict_clause():
+    rows = [{"project_id": "repo:a/b", "contracts": 2, "tier": "verified"}]
+    sql = format_insert_block(
+        "projects", ["project_id", "contracts", "tier"], rows, ["project_id"]
+    )
+    assert "INSERT INTO projects (project_id, contracts, tier) VALUES" in sql
+    assert "('repo:a/b', 2, 'verified')" in sql
+    assert "ON CONFLICT (project_id) DO NOTHING;" in sql