diff --git a/apps/api/src/hackproof/github/dependencies.py b/apps/api/src/hackproof/github/dependencies.py index 91ded2b..6b9c5ee 100644 --- a/apps/api/src/hackproof/github/dependencies.py +++ b/apps/api/src/hackproof/github/dependencies.py @@ -6,6 +6,7 @@ from hackproof.github.models import DependencyFingerprint MANIFESTS = {"Cargo.toml", "package.json", "requirements.txt", "pyproject.toml", "go.mod"} +SKIP_DIRECTORIES = {"node_modules", "dist", "target", ".git", "build", "vendor", ".next"} ECOSYSTEM = { "Cargo.toml": "rust", "package.json": "js", diff --git a/apps/api/src/hackproof/github/history.py b/apps/api/src/hackproof/github/history.py index 3dd70c4..55f857c 100644 --- a/apps/api/src/hackproof/github/history.py +++ b/apps/api/src/hackproof/github/history.py @@ -1,24 +1,47 @@ import asyncio +from datetime import datetime from pathlib import PurePosixPath from hackproof.github.client import GithubClient -from hackproof.github.dependencies import MANIFESTS, fingerprint +from hackproof.github.dependencies import MANIFESTS, SKIP_DIRECTORIES, fingerprint + +MAX_MANIFESTS = 60 + + +async def _stellar_packages_at( + github: GithubClient, full_name: str, sha: str, wanted: dict[str, list[str]] +) -> dict[str, str | None]: + tree = await github.json(f"/repos/{full_name}/git/trees/{sha}", {"recursive": "1"}) + paths = sorted( + ( + entry["path"] + for entry in (tree or {}).get("tree", []) + if entry["type"] == "blob" + and PurePosixPath(entry["path"]).name in MANIFESTS + and not SKIP_DIRECTORIES & set(PurePosixPath(entry["path"]).parts[:-1]) + ), + key=lambda path: (len(PurePosixPath(path).parts), path), + )[:MAX_MANIFESTS] + contents = await asyncio.gather(*(github.raw(full_name, sha, path) for path in paths)) + manifests = { + path: content.decode(errors="ignore") + for path, content in zip(paths, contents, strict=True) + if content + } + return fingerprint(manifests, wanted).packages async def stellar_added_in_window( github: GithubClient, full_name: str, - files: dict[str, str], wanted: dict[str, list[str]], base_sha: str | None, + window_end: datetime, ) -> bool: - if base_sha is None: - return True - manifests = [path for path in files if PurePosixPath(path).name in MANIFESTS] - contents = await asyncio.gather(*(github.raw(full_name, base_sha, path) for path in manifests)) - earlier = { - path: content.decode(errors="ignore") - for path, content in zip(manifests, contents, strict=True) - if content - } - return not fingerprint(earlier, wanted).packages + until = window_end.strftime("%Y-%m-%dT%H:%M:%SZ") + commits = await github.json(f"/repos/{full_name}/commits", {"until": until, "per_page": 1}) + if not commits: + return False + before = await _stellar_packages_at(github, full_name, base_sha, wanted) if base_sha else {} + after = await _stellar_packages_at(github, full_name, commits[0]["sha"], wanted) + return not before and bool(after) diff --git a/apps/api/src/hackproof/github/jobs.py b/apps/api/src/hackproof/github/jobs.py index c1d5a36..27860ee 100644 --- a/apps/api/src/hackproof/github/jobs.py +++ b/apps/api/src/hackproof/github/jobs.py @@ -78,7 +78,7 @@ async def run_scan(force: bool, projects_only: bool = False) -> None: repos = [Candidate(**row) for row in rows] log.info("scanning %s repos", len(repos)) - scanner = Scanner(github, http, hackathon.discovery) + scanner = Scanner(github, http, hackathon.discovery, hackathon.window) gate = asyncio.Semaphore(SCAN_CONCURRENCY) done = 0 diff --git a/apps/api/src/hackproof/github/scan.py b/apps/api/src/hackproof/github/scan.py index 2256ecd..ed251ae 100644 --- a/apps/api/src/hackproof/github/scan.py +++ b/apps/api/src/hackproof/github/scan.py @@ -8,13 +8,13 @@ from hackproof.github.client import GithubClient from hackproof.github.demo import scan_demo -from hackproof.github.dependencies import MANIFESTS, fingerprint +from hackproof.github.dependencies import MANIFESTS, SKIP_DIRECTORIES, fingerprint from hackproof.github.files import fetch_texts from hackproof.github.history import stellar_added_in_window from hackproof.github.models import Candidate, Contributor, Ref, RepoScan from hackproof.github.refs import extract_refs from hackproof.github.track import declared_track -from hackproof.hackathon import Discovery +from hackproof.hackathon import Discovery, Window from hackproof.integrations.catalog import Catalog, get_catalog from hackproof.integrations.detect import detect, detect_tools @@ -22,7 +22,6 @@ ".rs", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".toml", ".json", ".md", ".yml", ".yaml", ".sh", ".txt", ".html", ".env", ".sol", ".vue", ".svelte", } # fmt: skip -SKIP_DIRECTORIES = {"node_modules", "dist", "target", ".git", "build", "vendor", ".next"} SKIP_FILES = {"package-lock.json", "pnpm-lock.yaml", "yarn.lock", "Cargo.lock", "poetry.lock"} FILE_CONCURRENCY = 100 MAX_TEXT_FILES = 60 @@ -79,11 +78,13 @@ def __init__( github: GithubClient, http: httpx.AsyncClient, discovery: Discovery, + window: Window, catalog: Catalog | None = None, ) -> None: self._github = github self._http = http self._discovery = discovery + self._window = window self._catalog = catalog or get_catalog() self._files = asyncio.Semaphore(FILE_CONCURRENCY) @@ -118,7 +119,7 @@ async def scan(self, repo: Candidate) -> RepoScan | None: dependencies = fingerprint(files, wanted) added = ( await stellar_added_in_window( - self._github, repo.full_name, files, wanted, repo.base_sha + self._github, repo.full_name, wanted, repo.base_sha, self._window.end ) if dependencies.packages else None diff --git a/apps/api/tests/test_history.py b/apps/api/tests/test_history.py index 4afd572..d8c689f 100644 --- a/apps/api/tests/test_history.py +++ b/apps/api/tests/test_history.py @@ -1,34 +1,50 @@ +from datetime import UTC, datetime + from hackproof.github.history import stellar_added_in_window WANTED = {"js": ["@stellar/stellar-sdk"]} -HEAD = {"package.json": '{"dependencies": {"@stellar/stellar-sdk": "14.0.0"}}'} +END = datetime(2026, 10, 26, 12, tzinfo=UTC) +WITH = b'{"dependencies": {"@stellar/stellar-sdk": "14.0.0"}}' +WITHOUT = b'{"dependencies": {"react": "19"}}' class FakeGithub: - def __init__(self, earlier: dict[str, bytes]) -> None: - self._earlier = earlier + def __init__(self, commits: list[str], repo: dict[str, dict[str, bytes]]) -> None: + self._commits = commits + self._repo = repo + + async def json(self, path: str, params: dict | None = None): + if path.endswith("/commits"): + return [{"sha": sha} for sha in self._commits] + sha = path.rsplit("/", 1)[-1] + return {"tree": [{"type": "blob", "path": name} for name in self._repo.get(sha, {})]} async def raw(self, full_name: str, ref: str, path: str) -> bytes | None: - return self._earlier.get(path) + return self._repo.get(ref, {}).get(path) + + +async def added(base_sha: str | None, commits: list[str], repo: dict) -> bool: + return await stellar_added_in_window(FakeGithub(commits, repo), "a/b", WANTED, base_sha, END) -async def added(base_sha: str | None, earlier: dict[str, bytes]) -> bool: - return await stellar_added_in_window(FakeGithub(earlier), "a/b", HEAD, WANTED, base_sha) +async def test_stellar_missing_at_the_window_start_and_present_at_its_end_was_added_inside_it(): + repo = {"start": {"package.json": WITHOUT}, "end": {"package.json": WITH}} + assert await added("start", ["end"], repo) async def test_a_repo_with_no_history_before_the_window_was_born_inside_it(): - assert await added(None, {}) + assert await added(None, ["end"], {"end": {"package.json": WITH}}) -async def test_a_dependency_missing_before_the_window_was_added_during_it(): - before = {"package.json": b'{"dependencies": {"react": "19"}}'} - assert await added("abc", before) +async def test_stellar_already_present_at_the_window_start_is_not_new(): + repo = {"start": {"package.json": WITH}, "end": {"package.json": WITH}} + assert not await added("start", ["end"], repo) -async def test_a_manifest_that_did_not_exist_before_counts_as_added(): - assert await added("abc", {}) +async def test_stellar_added_only_after_the_window_ended_does_not_count(): + repo = {"start": {"package.json": WITHOUT}, "end": {"package.json": WITHOUT}} + assert not await added("start", ["end"], repo) -async def test_a_dependency_already_present_before_the_window_is_not_new(): - before = {"package.json": b'{"dependencies": {"@stellar/stellar-sdk": "13.0.0"}}'} - assert not await added("abc", before) +async def test_a_repo_with_no_commit_by_the_window_end_does_not_count(): + assert not await added(None, [], {})