diff --git a/.github/workflows/skill-source-drift.yml b/.github/workflows/skill-source-drift.yml new file mode 100644 index 000000000..ee5ba0563 --- /dev/null +++ b/.github/workflows/skill-source-drift.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Checks the curated skill sources in `organizations/*/skill-sources.md` +# against what their repositories ship today — see +# `tools/dev/check-skill-source-drift.py`. A source adding a skill, re-laying +# one out as a directory, tagging one for the first time, or tagging a newer +# version of a pinned one changes nothing in this repository, so without this +# the catalogue and the pins go stale unnoticed. +# +# Weekly (Monday), and from the Actions tab (*Run workflow*). +# +# ONE ISSUE, UPDATED IN PLACE. On drift, the open issue with the fixed title +# below gets the new report as its body, or one is opened. When everything is +# back in sync, that issue is closed with a comment. Nothing here edits the +# catalogue or moves a pin: both need a human to read the source first, and a +# moved tag in particular is a reason to look, not to re-pin. +--- +name: skill-source drift + +on: # yamllint disable-line rule:truthy + schedule: + - cron: "41 5 * * 1" + workflow_dispatch: + +permissions: {} + +env: + TITLE: "Skill-source drift: curated sources disagree with upstream" + +jobs: + check: + name: Check curated skill sources + runs-on: ubuntu-slim + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 1 + + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + enable-cache: true + cache-suffix: "skill-source-drift" + + - name: Compare the catalogues and pins with their sources + id: drift + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + uv run --project tools/dev python tools/dev/check-skill-source-drift.py \ + > "$RUNNER_TEMP/report.md" + status=$? + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + # 0 in sync, 1 drift, anything else: the check itself failed. + if [ "$status" -gt 1 ]; then + exit "$status" + fi + echo "drift=$status" >> "$GITHUB_OUTPUT" + + - name: Open, update or close the drift issue + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + DRIFT: ${{ steps.drift.outputs.drift }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + existing=$(gh issue list --repo "$REPO" --state open --search "in:title \"$TITLE\"" \ + --json number,title --jq ".[] | select(.title == env.TITLE) | .number" | head -n1) + + if [ "$DRIFT" = "0" ]; then + if [ -n "$existing" ]; then + gh issue close "$existing" --repo "$REPO" \ + --comment "Back in sync as of $RUN_URL." + fi + exit 0 + fi + + { + cat "$RUNNER_TEMP/report.md" + echo + echo "---" + echo + echo "Found by the weekly [skill-source drift]($RUN_URL) check." + echo "To resolve: read the change in the source repository, then update the catalogue" + echo "and, where a skill is ready, the descriptor's \`ref\`, \`commit\` and \`provides\`" + echo "in \`organizations//skill-sources.md\` (and \`docs/skill-sources/registry.md\`)." + echo "This issue closes itself once the next run finds everything in sync." + } > "$RUNNER_TEMP/body.md" + + if [ -n "$existing" ]; then + gh issue edit "$existing" --repo "$REPO" --body-file "$RUNNER_TEMP/body.md" + else + gh issue create --repo "$REPO" --title "$TITLE" \ + --label "family:setup" --label "family:ci" --body-file "$RUNNER_TEMP/body.md" + fi diff --git a/docs/skill-sources/registry.md b/docs/skill-sources/registry.md index b24ebef8b..a07882f51 100644 --- a/docs/skill-sources/registry.md +++ b/docs/skill-sources/registry.md @@ -38,9 +38,15 @@ Sources an organization vouches for, declared in `organization: `. Curation is still not installation — the adopter opts each one in. +An org file may also carry a *catalogue* of everything a source ships, pinned or not, +between `` markers. +The [`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) workflow +checks every catalogue and pin against its source each week +and opens an issue when they disagree, so a new or retagged skill upstream does not go unnoticed. + | Organization | Curated sources | File | |---|---|---| -| Apache Software Foundation | `apache-incubator`: [`apache/incubator`](https://github.com/apache/incubator) (`releasecheck`) | [`organizations/ASF/skill-sources.md`](../../organizations/ASF/skill-sources.md) | +| Apache Software Foundation | `apache-incubator`: [`apache/incubator`](https://github.com/apache/incubator) — pinned: `releasecheck`; also catalogued: the IPMC, mentor and podling skills in `tools/skills/` (graduation, retirement, report review, release-vote review, maturity model, …), with how each installs today ([catalogue](../../organizations/ASF/skill-sources.md#what-the-incubator-ships)) | [`organizations/ASF/skill-sources.md`](../../organizations/ASF/skill-sources.md) | | Independent (no formal governing body) | *(none listed yet)* | [`organizations/independent/skill-sources.md`](../../organizations/independent/skill-sources.md) | ## Community / external sources diff --git a/organizations/ASF/skill-sources.md b/organizations/ASF/skill-sources.md index 82bb6097d..a1f2ab1c2 100644 --- a/organizations/ASF/skill-sources.md +++ b/organizations/ASF/skill-sources.md @@ -28,15 +28,93 @@ table in [`docs/skill-sources/registry.md`](../../docs/skill-sources/registry.md ### Apache Incubator Skills the Incubator PMC maintains in -[`apache/incubator`](https://github.com/apache/incubator) for podlings, -mentors and the IPMC. The first is `releasecheck`, which checks a podling -release candidate before or during its vote. The same repo is also a plugin -marketplace, so the skills install without Magpie too: -`claude plugin marketplace add apache/incubator`. - -The source is pinned to a tag, because the rest of the Incubator repo -changes often. A new skill version gets a new `releasecheck-` tag, -and this pin moves only when the tag is updated here. +[`apache/incubator`](https://github.com/apache/incubator) under +[`tools/skills/`](https://github.com/apache/incubator/tree/HEAD/tools/skills) +for podlings, mentors and the IPMC. +There are two ways to install them, and they do not conflict: + +- **Through Magpie, for any agent.** + A project that trusts the source gets each pinned skill with + `/magpie-setup:setup skill-sources`, wired into `.agents/skills/` and relayed + to every agent's skills directory like a framework skill. + This is the path for an adopted repo, and the only one for agents with no + marketplace (OpenCode, Pi, Goose). +- **From the Incubator's own marketplaces, without Magpie.** + The repo carries a plugin catalog per agent; the *Marketplaces* column below + says which ones list a skill: + + | Marketplace | Add it | Catalog in `apache/incubator` | + |---|---|---| + | `claude-code` | `/plugin marketplace add apache/incubator` | `.claude-plugin/marketplace.json` | + | `codex` | `codex plugin marketplace add apache/incubator` | `.agents/plugins/marketplace.json` | + | `copilot` | add `apache/incubator` as a VS Code / GitHub Copilot plugin marketplace | `marketplace.json` | + + Updates then arrive through the agent's own plugin update. + +**Pinning and updates.** +The Incubator versions each skill on its own, tagging it `-` +(for example `releasecheck-0.5.1`), so each skill is curated as its own +descriptor pinned to its own tag: `apache-incubator` for `releasecheck`, and +`apache-incubator-` for each skill pinned after it. +A skill kept as a plugin directory (`tools/skills//skills//SKILL.md`, +the layout Codex needs to find it) is pinned with +`skills_root: tools/skills//skills`; the drift check still scans +`tools/skills/` for its siblings. +A pin moves only when it is updated here, and an adopter's committed pin moves +only when the adopter bumps it; `/magpie-setup:setup skill-sources` then +re-fetches and re-verifies the new tag. + +#### What the Incubator ships + +Every skill in `tools/skills/`, and how it can be installed today. +Most of these skills read the Incubator's own MCP servers (reports, podlings, +health, mail archives); each skill names the ones it needs, and the +[Incubator skills README](https://github.com/apache/incubator/blob/HEAD/tools/skills/README.md) +says how to set them up. + + +| Skill | What it does | Install through Magpie | Marketplaces | +|---|---|---|---| +| `releasecheck` | Checks a podling release candidate before or during its vote; findings for a person to review, never a vote. | `pinned` | `claude-code`, `codex` | +| `maturity-model-assessment` | Builds an evidence pack against the 38 items of the Apache Project Maturity Model, marking each evidenced, proxy or human; never a score. | `untagged` | — | +| `graduation-self-assessment` | Compares the `ipmc` MCP's graduation-readiness verdict with what the podling says in its own reports, checked against the mail archives. | `single-file` | — | +| `graduation-packet-builder` | Assembles a podling's graduation case and drafts the DISCUSS email and the board resolution. | `single-file` | — | +| `retirement-shepherd` | Assesses whether a stalled podling should retire and drafts the discussion, the announcement and the shutdown checklist. | `single-file` | — | +| `monthly-report-review` | The monthly IPMC report review: who has not filed, narratives against health data, draft shepherd comments. | `single-file` | — | +| `release-vote-reviewer` | Walks a release vote thread on general@ and drafts the +1 or the -1 with reasons. | `single-file` | — | +| `board-report` | Drafts the VP Incubator's quarterly report to the ASF board. | `single-file` | — | +| `branding-sweep` | Trademark and branding check across all current podlings, as a ranked fix list. | `single-file` | — | +| `podling-onboarding` | Reviews a new podling's proposal against the checklist and bootstraps its tracking. | `single-file` | — | +| `podling-name-clearance` | Runs the podling name search end to end and writes it up in the required format. | `single-file` | — | +| `asf-list-archaeology` | Not yet published (kept in `tools/skills/draft/`). | `draft` | — | + + +What *Install through Magpie* means: + +- **`pinned`** — in a descriptor's `provides` below. + An adopter that trusts that descriptor gets it with + `/magpie-setup:setup skill-sources`. +- **`untagged`** — already a `/SKILL.md` directory, which is the layout + Magpie fetches, but there is no release tag to pin yet. + It moves to `pinned` once the Incubator tags it. +- **`single-file`** — a single `.md` with its supporting files kept in + shared `scripts/`, `templates/` and `references/` directories. + Magpie wires a skill directory, and a marketplace installs one, so neither + can install these as they are; moving each into its own directory, as + `releasecheck` already is, is proposed to the Incubator. + Until then, build one with the Incubator's own packager and unpack it: + `python3 tools/skills/package-skills.py ` in an `apache/incubator` + checkout, then `unzip dist/.skill -d ~/.claude/skills/`. +- **`draft`** — not published by the Incubator; listed so it is not missed + when it is. + +The table is checked against `apache/incubator` every week by the +[`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) +workflow, which runs +[`tools/dev/check-skill-source-drift.py`](../../tools/dev/check-skill-source-drift.py) +and opens an issue when a skill was added, removed, re-laid-out or tagged, +when a marketplace catalog gained or lost it, or when a newer tag exists for a +pin — which is when a maintainer moves the pin here. **Podlings: prefer `releasecheck` over `release-verify-rc`.** Magpie's `release-verify-rc` does not distinguish a podling release from a TLP one, so diff --git a/tools/dev/README.md b/tools/dev/README.md index b7fd77948..265548577 100644 --- a/tools/dev/README.md +++ b/tools/dev/README.md @@ -64,6 +64,7 @@ installable for other members to depend on it. | [`check-family-plugins.py`](check-family-plugins.py) | Validates the marketplace plugins against the skills' `family:` frontmatter — version parity across every ecosystem manifest, Agent Plugins 1.0 conformance, and one well-formed per-family plugin whose `skills/` symlinks match the family exactly. `--fix` regenerates them, which is how the prek hook runs it. | | [`check-claude-mods.sh`](check-claude-mods.sh) | Runs `claude plugin validate --strict` and `claude plugin test` on every plugin whose `hooks/hooks.json` declares `modules` — the Claude Code mods ([`docs/when-to-use-mods.md`](../../docs/when-to-use-mods.md)). Claude Code skips a hooks module that fails its static analysis, so without this a mod that cannot load passes every other check and does nothing. A mod with no `*.test.ts` fails. Runs as the `check-claude-mods` prek hook, whose `language: node` env carries a pinned Claude Code. The hook is `stages: [manual]`, so commits never fetch the ~200 MB binary: the `prek` workflow runs it in a step of its own, and locally it is `prek run check-claude-mods --hook-stage manual --all-files`. A no-op when the tree has no mods. | | [`bump-hook-npm-pins.py`](bump-hook-npm-pins.py) | Moves the npm packages pinned in `.pre-commit-config.yaml` hooks' `additional_dependencies` — today the Claude Code CLI behind `check-claude-mods` — to the newest stable release past the package's cooldown: 12 hours for Claude Code, which ships several times a day and replaces a bad release within hours, 7 days otherwise, matching `.github/dependabot.yml`. Claude Code is a dev-only tool here — CI runs it to check the mods, and nothing from it ships. Dependabot's `pre-commit` ecosystem bumps a hook repo's `rev` but never these pins. Never moves a pin backwards. `--check` reports without writing. Run three times a week by [`bump-hook-npm-pins.yml`](../../.github/workflows/bump-hook-npm-pins.yml), which keeps a single draft PR open and updates it in place until it is merged. | +| [`check-skill-source-drift.py`](check-skill-source-drift.py) | Checks every curated skill source in `organizations/*/skill-sources.md` against what its repository ships today, through the GitHub API: the pinned tag still resolves to the recorded `commit`, every skill in `provides` is a `/SKILL.md` directory at that tag, no newer tag exists for the pin, and the source's *catalogue* (the table between `BEGIN`/`END SKILL-SOURCE CATALOGUE` markers) lists every skill the source ships with the status its layout gives it (`pinned`, `untagged`, `single-file`, `draft`) and the right marketplace column. A source adding, re-laying out or tagging a skill changes nothing here, so without it the catalogue goes stale unnoticed. Network-bound, so not a prek hook: the weekly [`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) workflow runs it and keeps one issue open while there is drift. Exit 0 in sync, 1 drift, 2 the check could not run. Read-only. | | [`bump-dev-version.py`](bump-dev-version.py) | Moves the `.dev` stamp on `project.version` in the root `pyproject.toml` — the single authority every manifest mirrors. Only the edit: `check-family-plugins.py --fix` and `uv lock` still follow, so the three steps read the same whether a human or CI runs them. The stamp is UTC at minute resolution and has to *move* for adopters to pick anything up (`claude plugin update` compares version strings, so a frozen suffix is a silent no-op). Refuses a version with no `.dev` suffix rather than stamping a release. Called by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`gh-signed-commit.py`](gh-signed-commit.py) | Commits the working tree through GitHub's `createCommitOnBranch` mutation instead of `git commit` + `git push`, so the commit is signed by GitHub and shows as **Verified** with no key material in CI. Collects changed and deleted paths from `git status --porcelain -z`, decomposes renames (the mutation has no rename concept), and pins `expectedHeadOid` so a concurrent push fails the call rather than being overwritten. Used by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`check-placeholders.sh`](check-placeholders.sh) | Fails the build on hardcoded project references in skill and tool docs, which must use `` / `` / `` / `` instead. Carries both casings and matches spaced variants. | diff --git a/tools/dev/check-skill-source-drift.py b/tools/dev/check-skill-source-drift.py new file mode 100644 index 000000000..174fc8bf2 --- /dev/null +++ b/tools/dev/check-skill-source-drift.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Check the curated skill sources against what their repositories ship today. + +A source that versions its skills independently is curated as one descriptor +per skill, each pinned to that skill's ``-`` tag. Descriptors +that share a repository are checked together, against one catalogue named +after any one of them. + +An organization's ``organizations//skill-sources.md`` pins a source to a +tag and may also carry a *catalogue*: a table, between +```` and the matching ``END`` +marker, of every skill the source ships and how each one installs. Both go +stale silently: a source adds a skill, turns a single-file skill into a +directory, tags a skill for the first time, or tags a newer version of the +pinned one, and nothing in this repository notices. This script notices. + +For every descriptor whose ``url`` is a GitHub repository it reads, through +the GitHub REST API: + +- the skills under ``layout.skills_root`` on the default branch: a ``.md`` + file (``single-file``), a ``/SKILL.md`` directory (``untagged``, or + ``pinned`` when ``provides`` names it), and anything under ``draft/`` + (``draft``); +- the repository's tags, to resolve the pin and find newer ones; +- the plugin catalog of each agent marketplace the source may publish to + (``MARKETPLACES``: Claude Code, Codex, GitHub Copilot), for the catalogue's + last column — the marketplaces a skill installs from, comma-separated, or + ``—`` for none. + +It reports, as Markdown on stdout: + +- a skill the source ships that the catalogue does not list, or the reverse; +- a catalogue status that no longer matches the source's layout; +- a skill named in ``provides`` that is not a directory at the pinned tag; +- a pinned tag that no longer resolves to the recorded ``commit`` (a moved + tag: treat it as suspicious, never re-pin blindly); +- a newer ``-`` tag for a pinned skill, or a first tag for an + unpinned one; +- a marketplaces column that disagrees with the source's catalogs. + +Exit status: 0 when everything matches, 1 when there is drift to act on, +2 when the check itself could not run (a network or API error, a catalogue +naming an unknown source). Read-only: it never edits a file. + +Usage: + + check-skill-source-drift.py # report on every org's sources + check-skill-source-drift.py --root DIR # repository root (default: this repo) + +Set ``GITHUB_TOKEN`` (or ``GH_TOKEN``) to raise the API rate limit. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +API = "https://api.github.com" + +STATUSES = ("pinned", "untagged", "single-file", "draft") +# Agent marketplace -> the catalog file it reads at the source's root. The same +# paths apache/magpie publishes its own plugins under (docs/setup/marketplace.md). +MARKETPLACES = { + "claude-code": ".claude-plugin/marketplace.json", + "codex": ".agents/plugins/marketplace.json", + "copilot": "marketplace.json", +} +DRAFT_DIR = "draft" + +_CATALOGUE_RE = re.compile( + r"(?P.*?)" + r"", + re.DOTALL, +) +_FENCE_RE = re.compile(r"^```ya?ml\s*$(?P.*?)^```\s*$", re.DOTALL | re.MULTILINE) +_GITHUB_URL_RE = re.compile(r"^https://github\.com/(?P[\w.-]+)/(?P[\w.-]+?)(?:\.git)?/?$") +_VERSIONED_REF_RE = re.compile(r"^(?P.+?)-v?(?P\d+(?:\.\d+)*)$") + + +class CheckError(Exception): + """The check could not run; distinct from drift, which is a result.""" + + +# -------------------------------------------------------------------------- +# Parsing the org files +# -------------------------------------------------------------------------- + + +@dataclass +class Descriptor: + id: str + url: str = "" + ref: str = "" + commit: str = "" + method: str = "" + skills_root: str = "skills" + provides: list[str] = field(default_factory=list) + + +@dataclass +class CatalogueRow: + skill: str + status: str + marketplaces: frozenset[str] + + +def parse_descriptors(text: str) -> list[Descriptor]: + """Read the descriptors in the ```yaml fences — stdlib only, the same + flat shape the skill validator parses. Commented lines and placeholder + ids (````) declare nothing.""" + out: list[Descriptor] = [] + for fence in _FENCE_RE.finditer(text): + cur: Descriptor | None = None + for raw in fence.group("body").splitlines(): + line = raw.rstrip() + if not line.strip() or line.lstrip().startswith("#"): + continue + m = re.match(r"^\s*(?:-\s+)?id:\s*(\S+)\s*$", line) + if m: + source_id = m.group(1).strip("'\"") + cur = None if "<" in source_id else Descriptor(id=source_id) + if cur is not None: + out.append(cur) + continue + if cur is None: + continue + m = re.match(r"^\s*-\s+skill:\s*(\S+)\s*$", line) + if m: + cur.provides.append(m.group(1).strip("'\"")) + continue + m = re.match(r"^\s*([A-Za-z_][\w-]*):\s*(.*?)\s*$", line) + if m: + key, val = m.group(1), m.group(2).split(" #")[0].strip().strip("'\"") + if key in ("url", "ref", "commit", "method", "skills_root") and val: + setattr(cur, key, val) + return out + + +def _cell(text: str) -> str: + return text.strip().strip("`").strip() + + +def parse_catalogues(text: str) -> dict[str, list[CatalogueRow]]: + """Read every catalogue table: first column the skill, the *Install + through Magpie* status the first cell that is a known status, the last + column the marketplaces the skill installs from.""" + out: dict[str, list[CatalogueRow]] = {} + for m in _CATALOGUE_RE.finditer(text): + rows: list[CatalogueRow] = [] + for line in m.group("body").splitlines(): + line = line.strip() + if not line.startswith("|") or set(line) <= set("|-: "): + continue + cells = [_cell(c) for c in line.strip("|").split("|")] + if cells[0].lower() == "skill": + continue + status = next((c for c in cells[1:] if c in STATUSES), "") + if not status: + raise CheckError(f"catalogue {m.group('id')}: row for {cells[0]!r} has no status") + names = frozenset(n.strip().strip("`") for n in cells[-1].split(",")) - {"", "—", "-"} + unknown = names - set(MARKETPLACES) + if unknown: + raise CheckError( + f"catalogue {m.group('id')}: {cells[0]!r} names unknown marketplace(s) {sorted(unknown)}" + ) + rows.append(CatalogueRow(cells[0], status, names)) + out[m.group("id")] = rows + return out + + +# -------------------------------------------------------------------------- +# Reading the source repository +# -------------------------------------------------------------------------- + +Fetch = Callable[[str], object] +"""GET an API path (``/repos/...``); return parsed JSON, or None on 404.""" + + +def github_fetch(path: str) -> object: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + req = urllib.request.Request(API + path, headers={"Accept": "application/vnd.github+json"}) + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise CheckError(f"GET {path}: HTTP {exc.code}") from exc + except urllib.error.URLError as exc: + raise CheckError(f"GET {path}: {exc.reason}") from exc + + +@dataclass +class Upstream: + layout: dict[str, str] + """skill name -> ``single-file`` / ``directory`` / ``draft`` at the default branch.""" + tags: dict[str, str] + """tag name -> the commit it resolves to (resolved lazily; '' until then).""" + marketplaces: dict[str, set[str]] + """marketplace -> the plugin names its catalog lists.""" + + +def _listing(fetch: Fetch, base: str, path: str, ref: str | None = None) -> list[dict[str, str]]: + query = f"?ref={urllib.parse.quote(ref)}" if ref else "" + got = fetch(f"{base}/contents/{urllib.parse.quote(path)}{query}") + return got if isinstance(got, list) else [] + + +def read_layout(fetch: Fetch, base: str, skills_root: str) -> dict[str, str]: + layout: dict[str, str] = {} + for entry in _listing(fetch, base, skills_root): + name, kind = entry["name"], entry["type"] + if kind == "file" and name.endswith(".md") and name.lower() != "readme.md": + layout[name[:-3]] = "single-file" + elif kind == "dir" and name == DRAFT_DIR: + for sub in _listing(fetch, base, f"{skills_root}/{name}"): + if sub["type"] == "file" and sub["name"].endswith(".md"): + layout[sub["name"][:-3]] = "draft" + elif sub["type"] == "dir": + layout[sub["name"]] = "draft" + elif kind == "dir": + files = {e["name"] for e in _listing(fetch, base, f"{skills_root}/{name}")} + if "SKILL.md" in files: + layout[name] = "directory" + elif "skills" in files: + # A plugin directory: the skill sits in skills//, where + # plugin installers (Codex in particular) look for it. + nested = {e["name"] for e in _listing(fetch, base, f"{skills_root}/{name}/skills/{name}")} + if "SKILL.md" in nested: + layout[name] = "directory" + return layout + + +def scan_root(desc: Descriptor) -> str: + """Where to look for every skill the source ships. + + A descriptor for one skill kept as a plugin directory points its + ``skills_root`` at ``//skills``; its siblings live in + ````.""" + for skill in desc.provides: + suffix = f"/{skill}/skills" + if desc.skills_root.endswith(suffix): + return desc.skills_root.removesuffix(suffix) + return desc.skills_root + + +def read_tags(fetch: Fetch, base: str) -> dict[str, str]: + refs = fetch(f"{base}/git/matching-refs/tags/") or [] + assert isinstance(refs, list) + return {r["ref"].removeprefix("refs/tags/"): "" for r in refs} + + +def resolve_tag(fetch: Fetch, base: str, tag: str) -> str: + """The commit a tag points at, peeling an annotated tag.""" + ref = fetch(f"{base}/git/ref/tags/{urllib.parse.quote(tag)}") + if not isinstance(ref, dict): + return "" + obj = ref["object"] + while obj["type"] == "tag": + tag_obj = fetch(f"{base}/git/tags/{obj['sha']}") + assert isinstance(tag_obj, dict) + obj = tag_obj["object"] + return str(obj["sha"]) + + +def read_marketplaces(fetch: Fetch, base: str) -> dict[str, set[str]]: + out: dict[str, set[str]] = {} + for market, path in MARKETPLACES.items(): + got = fetch(f"{base}/contents/{path}") + if not isinstance(got, dict) or "content" not in got: + out[market] = set() + continue + data = json.loads(base64.b64decode(got["content"])) + out[market] = {p["name"] for p in data.get("plugins", [])} + return out + + +# -------------------------------------------------------------------------- +# Comparing +# -------------------------------------------------------------------------- + + +def _version(text: str) -> tuple[int, ...]: + return tuple(int(p) for p in text.split(".")) + + +def tags_for(skill: str, tags: dict[str, str]) -> list[tuple[tuple[int, ...], str]]: + """``-`` tags for one skill, oldest first.""" + found = [] + for tag in tags: + m = _VERSIONED_REF_RE.match(tag) + if m and m.group("name") == skill: + found.append((_version(m.group("version")), tag)) + return sorted(found) + + +def check_pin(desc: Descriptor, upstream: Upstream, pin_commit: str, pinned_dirs: set[str]) -> list[str]: + """Findings for one descriptor's pin: the tag, its layout, newer tags.""" + findings: list[str] = [] + if desc.method == "git-tag" and desc.commit and pin_commit and pin_commit != desc.commit: + findings.append( + f"**Pinned tag moved.** `{desc.id}`: `{desc.ref}` now resolves to `{pin_commit[:12]}`, " + f"not the recorded `{desc.commit[:12]}`. Find out why before re-pinning." + ) + elif desc.method == "git-tag" and not pin_commit: + findings.append(f"**Pinned tag missing.** `{desc.id}`: `{desc.ref}` no longer exists in the source.") + for skill in desc.provides: + if skill not in pinned_dirs: + findings.append( + f"`{skill}` is in `{desc.id}`'s `provides` but is not a " + f"`{desc.skills_root}/{skill}/SKILL.md` directory at `{desc.ref}`, " + f"so `/magpie-setup:setup skill-sources` cannot wire it." + ) + pin_name = _VERSIONED_REF_RE.match(desc.ref) + if pin_name: + skill_tags = tags_for(pin_name.group("name"), upstream.tags) + if skill_tags and skill_tags[-1][0] > _version(pin_name.group("version")): + findings.append(f"**Newer tag** `{skill_tags[-1][1]}` for `{desc.id}` (pinned to `{desc.ref}`).") + return findings + + +def compare_catalogue(rows: list[CatalogueRow], upstream: Upstream, provided: set[str]) -> list[str]: + """Findings for the catalogue against the source's default branch.""" + findings: list[str] = [] + listed = {r.skill: r for r in rows} + for skill in sorted(set(upstream.layout) | set(listed)): + row, kind = listed.get(skill), upstream.layout.get(skill) + if row is None: + findings.append(f"**New skill** `{skill}` ({kind}) is not in the catalogue.") + continue + if kind is None: + findings.append(f"`{skill}` is in the catalogue but the source no longer ships it.") + continue + if skill in provided: + expected = "pinned" + elif kind == "directory": + expected = "untagged" + else: + expected = kind + if row.status != expected: + findings.append(f"`{skill}` is catalogued `{row.status}`, but the source makes it `{expected}`.") + actual = {market for market, names in upstream.marketplaces.items() if skill in names} + if actual != row.marketplaces: + shown = ", ".join(sorted(actual)) or "none" + findings.append( + f"`{skill}` installs from these marketplaces: {shown}; the catalogue says " + f"{', '.join(sorted(row.marketplaces)) or 'none'}." + ) + skill_tags = tags_for(skill, upstream.tags) + if skill_tags and skill not in provided and kind != "draft": + findings.append( + f"**Tagged** `{skill}` is now tagged (`{skill_tags[-1][1]}`) and could be pinned." + ) + return findings + + +def check_repo(fetch: Fetch, descs: list[Descriptor], rows: list[CatalogueRow] | None) -> list[str]: + """Check every descriptor that points at one repository, and its catalogue. + + A source that versions its skills independently is curated as one + descriptor per skill, each pinned to that skill's own tag; they share the + repository, so they share one catalogue.""" + m = _GITHUB_URL_RE.match(descs[0].url) + if not m: + return [f"Skipped: `{descs[0].url}` is not a GitHub repository; check it by hand."] + base = f"/repos/{m.group('owner')}/{m.group('repo')}" + upstream = Upstream( + layout=read_layout(fetch, base, scan_root(descs[0])), + tags=read_tags(fetch, base), + marketplaces=read_marketplaces(fetch, base), + ) + findings: list[str] = [] + provided: set[str] = set() + for desc in descs: + pin_commit = resolve_tag(fetch, base, desc.ref) if desc.method == "git-tag" else "" + pinned_dirs = set() + for skill in desc.provides: + files = {e["name"] for e in _listing(fetch, base, f"{desc.skills_root}/{skill}", desc.ref)} + if "SKILL.md" in files: + pinned_dirs.add(skill) + provided.update(desc.provides) + findings += check_pin(desc, upstream, pin_commit, pinned_dirs) + if rows is None: + findings.append( + f"No catalogue for `{descs[0].url}`: add one so new skills in the source get noticed." + ) + else: + findings += compare_catalogue(rows, upstream, provided) + return findings + + +def run(root: Path, fetch: Fetch) -> tuple[int, str]: + sections: list[str] = [] + drift = False + for path in sorted((root / "organizations").glob("*/skill-sources.md")): + if path.parent.name.startswith("_"): + continue + rel = path.relative_to(root) + text = path.read_text(encoding="utf-8") + descriptors = {d.id: d for d in parse_descriptors(text)} + catalogues = parse_catalogues(text) + unknown = set(catalogues) - set(descriptors) + if unknown: + raise CheckError(f"{rel}: catalogue for unknown source(s) {sorted(unknown)}") + by_repo: dict[str, list[Descriptor]] = {} + for desc in descriptors.values(): + by_repo.setdefault(desc.url.rstrip("/").removesuffix(".git"), []).append(desc) + for descs in by_repo.values(): + ids = {d.id for d in descs} + named = [cid for cid in catalogues if cid in ids] + if len(named) > 1: + raise CheckError(f"{rel}: more than one catalogue for {descs[0].url}: {named}") + rows = catalogues[named[0]] if named else None + findings = check_repo(fetch, descs, rows) + title = f"### `{descs[0].url}` ({rel})" + if findings: + drift = drift or any(not f.startswith("Skipped:") for f in findings) + sections.append(title + "\n\n" + "\n".join(f"- {f}" for f in findings)) + else: + sections.append(title + "\n\nIn sync.") + report = "## Skill-source drift\n\n" + ("\n\n".join(sections) if sections else "No curated sources.") + return (1 if drift else 0), report + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--root", type=Path, default=ROOT, help="repository root") + args = parser.parse_args(argv) + try: + status, report = run(args.root, github_fetch) + except CheckError as exc: + print(f"check-skill-source-drift: {exc}", file=sys.stderr) + return 2 + sys.stdout.write(report) + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/dev/tests/test_check_skill_source_drift.py b/tools/dev/tests/test_check_skill_source_drift.py new file mode 100644 index 000000000..2957f721e --- /dev/null +++ b/tools/dev/tests/test_check_skill_source_drift.py @@ -0,0 +1,321 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for ``check-skill-source-drift.py``. + +The GitHub API is replaced by a dict of canned responses, so nothing here +touches the network. What matters: an in-sync catalogue passes, and each kind +of upstream change (a new skill, a removed one, a re-layout, a first tag, a +newer tag, a moved tag, a marketplace change) is reported as drift. +""" + +from __future__ import annotations + +import base64 +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[1] / "check-skill-source-drift.py" + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_skill_source_drift", _SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # dataclasses resolve their module through sys.modules. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +mod = _load() + +PIN = "a" * 40 +BASE = "/repos/apache/incubator" + +ORG_FILE = """\ +# Curated + + +| Skill | What it does | Install through Magpie | Marketplaces | +|---|---|---|---| +| `releasecheck` | Checks an RC. | `pinned` | `claude-code`, `codex` | +| `maturity` | Evidence pack. | `untagged` | — | +| `graduation` | Readiness. | `single-file` | — | +| `archaeology` | Draft. | `draft` | — | + + +```yaml +- id: apache-incubator + organization: ASF + name: "Apache Incubator skills" + method: git-tag + url: https://github.com/apache/incubator + ref: releasecheck-0.5.1 + commit: {pin} + layout: + skills_root: tools/skills + provides: + - skill: releasecheck +``` +""" + + +def _entries(*items: tuple[str, str]) -> list[dict[str, str]]: + return [{"name": n, "type": t} for n, t in items] + + +def _catalog(data: object) -> dict[str, str]: + return {"content": base64.b64encode(json.dumps(data).encode()).decode()} + + +def _responses() -> dict[str, object]: + market = {"plugins": [{"name": "releasecheck"}]} + return { + f"{BASE}/contents/tools/skills": _entries( + ("README.md", "file"), + ("graduation.md", "file"), + ("releasecheck", "dir"), + ("maturity", "dir"), + ("draft", "dir"), + ("evals", "dir"), + ), + f"{BASE}/contents/tools/skills/releasecheck": _entries(("SKILL.md", "file")), + f"{BASE}/contents/tools/skills/maturity": _entries(("SKILL.md", "file"), ("references", "dir")), + f"{BASE}/contents/tools/skills/draft": _entries(("archaeology.md", "file")), + f"{BASE}/contents/tools/skills/evals": _entries(("run-evals.sh", "file")), + f"{BASE}/contents/tools/skills/releasecheck?ref=releasecheck-0.5.1": _entries(("SKILL.md", "file")), + f"{BASE}/git/matching-refs/tags/": [ + {"ref": "refs/tags/releasecheck-0.5"}, + {"ref": "refs/tags/releasecheck-0.5.1"}, + {"ref": "refs/tags/releases/tools/1.0.0"}, + ], + f"{BASE}/git/ref/tags/releasecheck-0.5.1": {"object": {"type": "tag", "sha": "t1"}}, + f"{BASE}/git/tags/t1": {"object": {"type": "commit", "sha": PIN}}, + f"{BASE}/contents/.claude-plugin/marketplace.json": _catalog(market), + f"{BASE}/contents/.agents/plugins/marketplace.json": _catalog(market), + } + + +def _run(tmp_path: Path, responses: dict[str, object], org_file: str = ORG_FILE) -> tuple[int, str]: + org = tmp_path / "organizations" / "ASF" + org.mkdir(parents=True) + (org / "skill-sources.md").write_text(org_file.replace("{pin}", PIN), encoding="utf-8") + return mod.run(tmp_path, responses.get) + + +def test_in_sync(tmp_path: Path) -> None: + status, report = _run(tmp_path, _responses()) + assert status == 0, report + assert "In sync." in report + + +def test_new_skill_upstream(tmp_path: Path) -> None: + responses = _responses() + listing = responses[f"{BASE}/contents/tools/skills"] + assert isinstance(listing, list) + listing.append({"name": "board-report.md", "type": "file"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**New skill** `board-report` (single-file)" in report + + +def test_removed_skill(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/contents/tools/skills/draft"] = [] + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`archaeology` is in the catalogue but the source no longer ships it" in report + + +def test_single_file_became_directory(tmp_path: Path) -> None: + responses = _responses() + listing = responses[f"{BASE}/contents/tools/skills"] + assert isinstance(listing, list) + listing.remove({"name": "graduation.md", "type": "file"}) + listing.append({"name": "graduation", "type": "dir"}) + responses[f"{BASE}/contents/tools/skills/graduation"] = _entries(("SKILL.md", "file")) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`graduation` is catalogued `single-file`, but the source makes it `untagged`" in report + + +def test_first_tag_for_unpinned_skill(tmp_path: Path) -> None: + responses = _responses() + tags = responses[f"{BASE}/git/matching-refs/tags/"] + assert isinstance(tags, list) + tags.append({"ref": "refs/tags/maturity-0.1"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Tagged** `maturity` is now tagged (`maturity-0.1`)" in report + + +def test_newer_tag_for_pinned_skill(tmp_path: Path) -> None: + responses = _responses() + tags = responses[f"{BASE}/git/matching-refs/tags/"] + assert isinstance(tags, list) + tags.append({"ref": "refs/tags/releasecheck-0.10"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Newer tag** `releasecheck-0.10`" in report + + +def test_moved_tag(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/git/tags/t1"] = {"object": {"type": "commit", "sha": "b" * 40}} + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Pinned tag moved.**" in report + + +def test_missing_tag(tmp_path: Path) -> None: + responses = _responses() + del responses[f"{BASE}/git/ref/tags/releasecheck-0.5.1"] + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Pinned tag missing.**" in report + + +def test_marketplace_change(tmp_path: Path) -> None: + responses = _responses() + market = {"plugins": [{"name": "releasecheck"}, {"name": "maturity"}]} + responses[f"{BASE}/contents/marketplace.json"] = _catalog(market) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`maturity` installs from these marketplaces: copilot; the catalogue says none" in report + + +def test_unknown_marketplace_is_an_error(tmp_path: Path) -> None: + org_file = ORG_FILE.replace("`claude-code`, `codex` |", "`claude-code`, `nowhere` |") + with pytest.raises(mod.CheckError, match="unknown marketplace"): + _run(tmp_path, _responses(), org_file) + + +def test_provided_skill_not_a_directory_at_pin(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/contents/tools/skills/releasecheck?ref=releasecheck-0.5.1"] = None + status, report = _run(tmp_path, responses) + assert status == 1 + assert "is in `apache-incubator`'s `provides` but is not a" in report + + +def test_catalogue_without_descriptor_is_an_error(tmp_path: Path) -> None: + org_file = ORG_FILE.replace("CATALOGUE: apache-incubator", "CATALOGUE: elsewhere") + with pytest.raises(mod.CheckError, match="unknown source"): + _run(tmp_path, _responses(), org_file) + + +def test_descriptor_without_catalogue_is_drift(tmp_path: Path) -> None: + start = ORG_FILE.index("", ORG_FILE.index("