From 9742dc1b03c24d6d687dc980599529bf9e6dde39 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 9 Nov 2026 01:10:17 -0700 Subject: [PATCH 1/2] Catalogue the Incubator skills and check curated sources weekly The ASF organization curated apache/incubator for releasecheck only, while the Incubator now ships eleven skills (graduation self-assessment, maturity model, retirement, report review, and more). Nothing noticed. organizations/ASF/skill-sources.md now lists every skill the Incubator ships, whether Magpie can install it (pinned, untagged, single-file, draft) and which agent marketplaces offer it, and explains the two install paths: through Magpie for any agent, or from the Incubator's own Claude Code, Codex and Copilot catalogs. Skills are versioned per skill, so each one is curated as its own descriptor pinned to its own tag. tools/dev/check-skill-source-drift.py compares every curated catalogue and pin with its GitHub source: new, removed or re-laid-out skills, first or newer tags, moved tags, and marketplace catalog changes. The weekly skill-source-drift workflow runs it and keeps one issue open while there is drift. Generated-by: Claude Opus 5 --- .github/workflows/skill-source-drift.yml | 116 +++++ docs/skill-sources/registry.md | 8 +- organizations/ASF/skill-sources.md | 92 +++- tools/dev/README.md | 1 + tools/dev/check-skill-source-drift.py | 452 ++++++++++++++++++ .../tests/test_check_skill_source_drift.py | 271 +++++++++++ 6 files changed, 930 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/skill-source-drift.yml create mode 100644 tools/dev/check-skill-source-drift.py create mode 100644 tools/dev/tests/test_check_skill_source_drift.py diff --git a/.github/workflows/skill-source-drift.yml b/.github/workflows/skill-source-drift.yml new file mode 100644 index 000000000..ee5ba0563 --- /dev/null +++ b/.github/workflows/skill-source-drift.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Checks the curated skill sources in `organizations/*/skill-sources.md` +# against what their repositories ship today — see +# `tools/dev/check-skill-source-drift.py`. A source adding a skill, re-laying +# one out as a directory, tagging one for the first time, or tagging a newer +# version of a pinned one changes nothing in this repository, so without this +# the catalogue and the pins go stale unnoticed. +# +# Weekly (Monday), and from the Actions tab (*Run workflow*). +# +# ONE ISSUE, UPDATED IN PLACE. On drift, the open issue with the fixed title +# below gets the new report as its body, or one is opened. When everything is +# back in sync, that issue is closed with a comment. Nothing here edits the +# catalogue or moves a pin: both need a human to read the source first, and a +# moved tag in particular is a reason to look, not to re-pin. +--- +name: skill-source drift + +on: # yamllint disable-line rule:truthy + schedule: + - cron: "41 5 * * 1" + workflow_dispatch: + +permissions: {} + +env: + TITLE: "Skill-source drift: curated sources disagree with upstream" + +jobs: + check: + name: Check curated skill sources + runs-on: ubuntu-slim + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 1 + + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + enable-cache: true + cache-suffix: "skill-source-drift" + + - name: Compare the catalogues and pins with their sources + id: drift + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + uv run --project tools/dev python tools/dev/check-skill-source-drift.py \ + > "$RUNNER_TEMP/report.md" + status=$? + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + # 0 in sync, 1 drift, anything else: the check itself failed. + if [ "$status" -gt 1 ]; then + exit "$status" + fi + echo "drift=$status" >> "$GITHUB_OUTPUT" + + - name: Open, update or close the drift issue + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + DRIFT: ${{ steps.drift.outputs.drift }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + existing=$(gh issue list --repo "$REPO" --state open --search "in:title \"$TITLE\"" \ + --json number,title --jq ".[] | select(.title == env.TITLE) | .number" | head -n1) + + if [ "$DRIFT" = "0" ]; then + if [ -n "$existing" ]; then + gh issue close "$existing" --repo "$REPO" \ + --comment "Back in sync as of $RUN_URL." + fi + exit 0 + fi + + { + cat "$RUNNER_TEMP/report.md" + echo + echo "---" + echo + echo "Found by the weekly [skill-source drift]($RUN_URL) check." + echo "To resolve: read the change in the source repository, then update the catalogue" + echo "and, where a skill is ready, the descriptor's \`ref\`, \`commit\` and \`provides\`" + echo "in \`organizations//skill-sources.md\` (and \`docs/skill-sources/registry.md\`)." + echo "This issue closes itself once the next run finds everything in sync." + } > "$RUNNER_TEMP/body.md" + + if [ -n "$existing" ]; then + gh issue edit "$existing" --repo "$REPO" --body-file "$RUNNER_TEMP/body.md" + else + gh issue create --repo "$REPO" --title "$TITLE" \ + --label "family:setup" --label "family:ci" --body-file "$RUNNER_TEMP/body.md" + fi diff --git a/docs/skill-sources/registry.md b/docs/skill-sources/registry.md index b24ebef8b..a07882f51 100644 --- a/docs/skill-sources/registry.md +++ b/docs/skill-sources/registry.md @@ -38,9 +38,15 @@ Sources an organization vouches for, declared in `organization: `. Curation is still not installation — the adopter opts each one in. +An org file may also carry a *catalogue* of everything a source ships, pinned or not, +between `` markers. +The [`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) workflow +checks every catalogue and pin against its source each week +and opens an issue when they disagree, so a new or retagged skill upstream does not go unnoticed. + | Organization | Curated sources | File | |---|---|---| -| Apache Software Foundation | `apache-incubator`: [`apache/incubator`](https://github.com/apache/incubator) (`releasecheck`) | [`organizations/ASF/skill-sources.md`](../../organizations/ASF/skill-sources.md) | +| Apache Software Foundation | `apache-incubator`: [`apache/incubator`](https://github.com/apache/incubator) — pinned: `releasecheck`; also catalogued: the IPMC, mentor and podling skills in `tools/skills/` (graduation, retirement, report review, release-vote review, maturity model, …), with how each installs today ([catalogue](../../organizations/ASF/skill-sources.md#what-the-incubator-ships)) | [`organizations/ASF/skill-sources.md`](../../organizations/ASF/skill-sources.md) | | Independent (no formal governing body) | *(none listed yet)* | [`organizations/independent/skill-sources.md`](../../organizations/independent/skill-sources.md) | ## Community / external sources diff --git a/organizations/ASF/skill-sources.md b/organizations/ASF/skill-sources.md index 82bb6097d..fe0d20e08 100644 --- a/organizations/ASF/skill-sources.md +++ b/organizations/ASF/skill-sources.md @@ -28,15 +28,89 @@ table in [`docs/skill-sources/registry.md`](../../docs/skill-sources/registry.md ### Apache Incubator Skills the Incubator PMC maintains in -[`apache/incubator`](https://github.com/apache/incubator) for podlings, -mentors and the IPMC. The first is `releasecheck`, which checks a podling -release candidate before or during its vote. The same repo is also a plugin -marketplace, so the skills install without Magpie too: -`claude plugin marketplace add apache/incubator`. - -The source is pinned to a tag, because the rest of the Incubator repo -changes often. A new skill version gets a new `releasecheck-` tag, -and this pin moves only when the tag is updated here. +[`apache/incubator`](https://github.com/apache/incubator) under +[`tools/skills/`](https://github.com/apache/incubator/tree/HEAD/tools/skills) +for podlings, mentors and the IPMC. +There are two ways to install them, and they do not conflict: + +- **Through Magpie, for any agent.** + A project that trusts the source gets each pinned skill with + `/magpie-setup:setup skill-sources`, wired into `.agents/skills/` and relayed + to every agent's skills directory like a framework skill. + This is the path for an adopted repo, and the only one for agents with no + marketplace (OpenCode, Pi, Goose). +- **From the Incubator's own marketplaces, without Magpie.** + The repo carries a plugin catalog per agent; the *Marketplaces* column below + says which ones list a skill: + + | Marketplace | Add it | Catalog in `apache/incubator` | + |---|---|---| + | `claude-code` | `/plugin marketplace add apache/incubator` | `.claude-plugin/marketplace.json` | + | `codex` | `codex plugin marketplace add apache/incubator` | `.agents/plugins/marketplace.json` | + | `copilot` | add `apache/incubator` as a VS Code / GitHub Copilot plugin marketplace | `marketplace.json` | + + Updates then arrive through the agent's own plugin update. + +**Pinning and updates.** +The Incubator versions each skill on its own, tagging it `-` +(for example `releasecheck-0.5.1`), so each skill is curated as its own +descriptor pinned to its own tag: `apache-incubator` for `releasecheck`, and +`apache-incubator-` for each skill pinned after it. +A pin moves only when it is updated here, and an adopter's committed pin moves +only when the adopter bumps it; `/magpie-setup:setup skill-sources` then +re-fetches and re-verifies the new tag. + +#### What the Incubator ships + +Every skill in `tools/skills/`, and how it can be installed today. +Most of these skills read the Incubator's own MCP servers (reports, podlings, +health, mail archives); each skill names the ones it needs, and the +[Incubator skills README](https://github.com/apache/incubator/blob/HEAD/tools/skills/README.md) +says how to set them up. + + +| Skill | What it does | Install through Magpie | Marketplaces | +|---|---|---|---| +| `releasecheck` | Checks a podling release candidate before or during its vote; findings for a person to review, never a vote. | `pinned` | `claude-code`, `codex` | +| `maturity-model-assessment` | Builds an evidence pack against the 38 items of the Apache Project Maturity Model, marking each evidenced, proxy or human; never a score. | `untagged` | — | +| `graduation-self-assessment` | Compares the `ipmc` MCP's graduation-readiness verdict with what the podling says in its own reports, checked against the mail archives. | `single-file` | — | +| `graduation-packet-builder` | Assembles a podling's graduation case and drafts the DISCUSS email and the board resolution. | `single-file` | — | +| `retirement-shepherd` | Assesses whether a stalled podling should retire and drafts the discussion, the announcement and the shutdown checklist. | `single-file` | — | +| `monthly-report-review` | The monthly IPMC report review: who has not filed, narratives against health data, draft shepherd comments. | `single-file` | — | +| `release-vote-reviewer` | Walks a release vote thread on general@ and drafts the +1 or the -1 with reasons. | `single-file` | — | +| `board-report` | Drafts the VP Incubator's quarterly report to the ASF board. | `single-file` | — | +| `branding-sweep` | Trademark and branding check across all current podlings, as a ranked fix list. | `single-file` | — | +| `podling-onboarding` | Reviews a new podling's proposal against the checklist and bootstraps its tracking. | `single-file` | — | +| `podling-name-clearance` | Runs the podling name search end to end and writes it up in the required format. | `single-file` | — | +| `asf-list-archaeology` | Not yet published (kept in `tools/skills/draft/`). | `draft` | — | + + +What *Install through Magpie* means: + +- **`pinned`** — in a descriptor's `provides` below. + An adopter that trusts that descriptor gets it with + `/magpie-setup:setup skill-sources`. +- **`untagged`** — already a `/SKILL.md` directory, which is the layout + Magpie fetches, but there is no release tag to pin yet. + It moves to `pinned` once the Incubator tags it. +- **`single-file`** — a single `.md` with its supporting files kept in + shared `scripts/`, `templates/` and `references/` directories. + Magpie wires a skill directory, and a marketplace installs one, so neither + can install these as they are; moving each into its own directory, as + `releasecheck` already is, is proposed to the Incubator. + Until then, build one with the Incubator's own packager and unpack it: + `python3 tools/skills/package-skills.py ` in an `apache/incubator` + checkout, then `unzip dist/.skill -d ~/.claude/skills/`. +- **`draft`** — not published by the Incubator; listed so it is not missed + when it is. + +The table is checked against `apache/incubator` every week by the +[`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) +workflow, which runs +[`tools/dev/check-skill-source-drift.py`](../../tools/dev/check-skill-source-drift.py) +and opens an issue when a skill was added, removed, re-laid-out or tagged, +when a marketplace catalog gained or lost it, or when a newer tag exists for a +pin — which is when a maintainer moves the pin here. **Podlings: prefer `releasecheck` over `release-verify-rc`.** Magpie's `release-verify-rc` does not distinguish a podling release from a TLP one, so diff --git a/tools/dev/README.md b/tools/dev/README.md index b7fd77948..265548577 100644 --- a/tools/dev/README.md +++ b/tools/dev/README.md @@ -64,6 +64,7 @@ installable for other members to depend on it. | [`check-family-plugins.py`](check-family-plugins.py) | Validates the marketplace plugins against the skills' `family:` frontmatter — version parity across every ecosystem manifest, Agent Plugins 1.0 conformance, and one well-formed per-family plugin whose `skills/` symlinks match the family exactly. `--fix` regenerates them, which is how the prek hook runs it. | | [`check-claude-mods.sh`](check-claude-mods.sh) | Runs `claude plugin validate --strict` and `claude plugin test` on every plugin whose `hooks/hooks.json` declares `modules` — the Claude Code mods ([`docs/when-to-use-mods.md`](../../docs/when-to-use-mods.md)). Claude Code skips a hooks module that fails its static analysis, so without this a mod that cannot load passes every other check and does nothing. A mod with no `*.test.ts` fails. Runs as the `check-claude-mods` prek hook, whose `language: node` env carries a pinned Claude Code. The hook is `stages: [manual]`, so commits never fetch the ~200 MB binary: the `prek` workflow runs it in a step of its own, and locally it is `prek run check-claude-mods --hook-stage manual --all-files`. A no-op when the tree has no mods. | | [`bump-hook-npm-pins.py`](bump-hook-npm-pins.py) | Moves the npm packages pinned in `.pre-commit-config.yaml` hooks' `additional_dependencies` — today the Claude Code CLI behind `check-claude-mods` — to the newest stable release past the package's cooldown: 12 hours for Claude Code, which ships several times a day and replaces a bad release within hours, 7 days otherwise, matching `.github/dependabot.yml`. Claude Code is a dev-only tool here — CI runs it to check the mods, and nothing from it ships. Dependabot's `pre-commit` ecosystem bumps a hook repo's `rev` but never these pins. Never moves a pin backwards. `--check` reports without writing. Run three times a week by [`bump-hook-npm-pins.yml`](../../.github/workflows/bump-hook-npm-pins.yml), which keeps a single draft PR open and updates it in place until it is merged. | +| [`check-skill-source-drift.py`](check-skill-source-drift.py) | Checks every curated skill source in `organizations/*/skill-sources.md` against what its repository ships today, through the GitHub API: the pinned tag still resolves to the recorded `commit`, every skill in `provides` is a `/SKILL.md` directory at that tag, no newer tag exists for the pin, and the source's *catalogue* (the table between `BEGIN`/`END SKILL-SOURCE CATALOGUE` markers) lists every skill the source ships with the status its layout gives it (`pinned`, `untagged`, `single-file`, `draft`) and the right marketplace column. A source adding, re-laying out or tagging a skill changes nothing here, so without it the catalogue goes stale unnoticed. Network-bound, so not a prek hook: the weekly [`skill-source-drift`](../../.github/workflows/skill-source-drift.yml) workflow runs it and keeps one issue open while there is drift. Exit 0 in sync, 1 drift, 2 the check could not run. Read-only. | | [`bump-dev-version.py`](bump-dev-version.py) | Moves the `.dev` stamp on `project.version` in the root `pyproject.toml` — the single authority every manifest mirrors. Only the edit: `check-family-plugins.py --fix` and `uv lock` still follow, so the three steps read the same whether a human or CI runs them. The stamp is UTC at minute resolution and has to *move* for adopters to pick anything up (`claude plugin update` compares version strings, so a frozen suffix is a silent no-op). Refuses a version with no `.dev` suffix rather than stamping a release. Called by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`gh-signed-commit.py`](gh-signed-commit.py) | Commits the working tree through GitHub's `createCommitOnBranch` mutation instead of `git commit` + `git push`, so the commit is signed by GitHub and shows as **Verified** with no key material in CI. Collects changed and deleted paths from `git status --porcelain -z`, decomposes renames (the mutation has no rename concept), and pins `expectedHeadOid` so a concurrent push fails the call rather than being overwritten. Used by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`check-placeholders.sh`](check-placeholders.sh) | Fails the build on hardcoded project references in skill and tool docs, which must use `` / `` / `` / `` instead. Carries both casings and matches spaced variants. | diff --git a/tools/dev/check-skill-source-drift.py b/tools/dev/check-skill-source-drift.py new file mode 100644 index 000000000..8e1d5050a --- /dev/null +++ b/tools/dev/check-skill-source-drift.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Check the curated skill sources against what their repositories ship today. + +A source that versions its skills independently is curated as one descriptor +per skill, each pinned to that skill's ``-`` tag. Descriptors +that share a repository are checked together, against one catalogue named +after any one of them. + +An organization's ``organizations//skill-sources.md`` pins a source to a +tag and may also carry a *catalogue*: a table, between +```` and the matching ``END`` +marker, of every skill the source ships and how each one installs. Both go +stale silently: a source adds a skill, turns a single-file skill into a +directory, tags a skill for the first time, or tags a newer version of the +pinned one, and nothing in this repository notices. This script notices. + +For every descriptor whose ``url`` is a GitHub repository it reads, through +the GitHub REST API: + +- the skills under ``layout.skills_root`` on the default branch: a ``.md`` + file (``single-file``), a ``/SKILL.md`` directory (``untagged``, or + ``pinned`` when ``provides`` names it), and anything under ``draft/`` + (``draft``); +- the repository's tags, to resolve the pin and find newer ones; +- the plugin catalog of each agent marketplace the source may publish to + (``MARKETPLACES``: Claude Code, Codex, GitHub Copilot), for the catalogue's + last column — the marketplaces a skill installs from, comma-separated, or + ``—`` for none. + +It reports, as Markdown on stdout: + +- a skill the source ships that the catalogue does not list, or the reverse; +- a catalogue status that no longer matches the source's layout; +- a skill named in ``provides`` that is not a directory at the pinned tag; +- a pinned tag that no longer resolves to the recorded ``commit`` (a moved + tag: treat it as suspicious, never re-pin blindly); +- a newer ``-`` tag for a pinned skill, or a first tag for an + unpinned one; +- a marketplaces column that disagrees with the source's catalogs. + +Exit status: 0 when everything matches, 1 when there is drift to act on, +2 when the check itself could not run (a network or API error, a catalogue +naming an unknown source). Read-only: it never edits a file. + +Usage: + + check-skill-source-drift.py # report on every org's sources + check-skill-source-drift.py --root DIR # repository root (default: this repo) + +Set ``GITHUB_TOKEN`` (or ``GH_TOKEN``) to raise the API rate limit. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +API = "https://api.github.com" + +STATUSES = ("pinned", "untagged", "single-file", "draft") +# Agent marketplace -> the catalog file it reads at the source's root. The same +# paths apache/magpie publishes its own plugins under (docs/setup/marketplace.md). +MARKETPLACES = { + "claude-code": ".claude-plugin/marketplace.json", + "codex": ".agents/plugins/marketplace.json", + "copilot": "marketplace.json", +} +DRAFT_DIR = "draft" + +_CATALOGUE_RE = re.compile( + r"(?P.*?)" + r"", + re.DOTALL, +) +_FENCE_RE = re.compile(r"^```ya?ml\s*$(?P.*?)^```\s*$", re.DOTALL | re.MULTILINE) +_GITHUB_URL_RE = re.compile(r"^https://github\.com/(?P[\w.-]+)/(?P[\w.-]+?)(?:\.git)?/?$") +_VERSIONED_REF_RE = re.compile(r"^(?P.+?)-v?(?P\d+(?:\.\d+)*)$") + + +class CheckError(Exception): + """The check could not run; distinct from drift, which is a result.""" + + +# -------------------------------------------------------------------------- +# Parsing the org files +# -------------------------------------------------------------------------- + + +@dataclass +class Descriptor: + id: str + url: str = "" + ref: str = "" + commit: str = "" + method: str = "" + skills_root: str = "skills" + provides: list[str] = field(default_factory=list) + + +@dataclass +class CatalogueRow: + skill: str + status: str + marketplaces: frozenset[str] + + +def parse_descriptors(text: str) -> list[Descriptor]: + """Read the descriptors in the ```yaml fences — stdlib only, the same + flat shape the skill validator parses. Commented lines and placeholder + ids (````) declare nothing.""" + out: list[Descriptor] = [] + for fence in _FENCE_RE.finditer(text): + cur: Descriptor | None = None + for raw in fence.group("body").splitlines(): + line = raw.rstrip() + if not line.strip() or line.lstrip().startswith("#"): + continue + m = re.match(r"^\s*(?:-\s+)?id:\s*(\S+)\s*$", line) + if m: + source_id = m.group(1).strip("'\"") + cur = None if "<" in source_id else Descriptor(id=source_id) + if cur is not None: + out.append(cur) + continue + if cur is None: + continue + m = re.match(r"^\s*-\s+skill:\s*(\S+)\s*$", line) + if m: + cur.provides.append(m.group(1).strip("'\"")) + continue + m = re.match(r"^\s*([A-Za-z_][\w-]*):\s*(.*?)\s*$", line) + if m: + key, val = m.group(1), m.group(2).split(" #")[0].strip().strip("'\"") + if key in ("url", "ref", "commit", "method", "skills_root") and val: + setattr(cur, key, val) + return out + + +def _cell(text: str) -> str: + return text.strip().strip("`").strip() + + +def parse_catalogues(text: str) -> dict[str, list[CatalogueRow]]: + """Read every catalogue table: first column the skill, the *Install + through Magpie* status the first cell that is a known status, the last + column the marketplaces the skill installs from.""" + out: dict[str, list[CatalogueRow]] = {} + for m in _CATALOGUE_RE.finditer(text): + rows: list[CatalogueRow] = [] + for line in m.group("body").splitlines(): + line = line.strip() + if not line.startswith("|") or set(line) <= set("|-: "): + continue + cells = [_cell(c) for c in line.strip("|").split("|")] + if cells[0].lower() == "skill": + continue + status = next((c for c in cells[1:] if c in STATUSES), "") + if not status: + raise CheckError(f"catalogue {m.group('id')}: row for {cells[0]!r} has no status") + names = frozenset(n.strip().strip("`") for n in cells[-1].split(",")) - {"", "—", "-"} + unknown = names - set(MARKETPLACES) + if unknown: + raise CheckError( + f"catalogue {m.group('id')}: {cells[0]!r} names unknown marketplace(s) {sorted(unknown)}" + ) + rows.append(CatalogueRow(cells[0], status, names)) + out[m.group("id")] = rows + return out + + +# -------------------------------------------------------------------------- +# Reading the source repository +# -------------------------------------------------------------------------- + +Fetch = Callable[[str], object] +"""GET an API path (``/repos/...``); return parsed JSON, or None on 404.""" + + +def github_fetch(path: str) -> object: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + req = urllib.request.Request(API + path, headers={"Accept": "application/vnd.github+json"}) + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise CheckError(f"GET {path}: HTTP {exc.code}") from exc + except urllib.error.URLError as exc: + raise CheckError(f"GET {path}: {exc.reason}") from exc + + +@dataclass +class Upstream: + layout: dict[str, str] + """skill name -> ``single-file`` / ``directory`` / ``draft`` at the default branch.""" + tags: dict[str, str] + """tag name -> the commit it resolves to (resolved lazily; '' until then).""" + marketplaces: dict[str, set[str]] + """marketplace -> the plugin names its catalog lists.""" + + +def _listing(fetch: Fetch, base: str, path: str, ref: str | None = None) -> list[dict[str, str]]: + query = f"?ref={urllib.parse.quote(ref)}" if ref else "" + got = fetch(f"{base}/contents/{urllib.parse.quote(path)}{query}") + return got if isinstance(got, list) else [] + + +def read_layout(fetch: Fetch, base: str, skills_root: str) -> dict[str, str]: + layout: dict[str, str] = {} + for entry in _listing(fetch, base, skills_root): + name, kind = entry["name"], entry["type"] + if kind == "file" and name.endswith(".md") and name.lower() != "readme.md": + layout[name[:-3]] = "single-file" + elif kind == "dir" and name == DRAFT_DIR: + for sub in _listing(fetch, base, f"{skills_root}/{name}"): + if sub["type"] == "file" and sub["name"].endswith(".md"): + layout[sub["name"][:-3]] = "draft" + elif sub["type"] == "dir": + layout[sub["name"]] = "draft" + elif kind == "dir": + files = {e["name"] for e in _listing(fetch, base, f"{skills_root}/{name}")} + if "SKILL.md" in files: + layout[name] = "directory" + return layout + + +def read_tags(fetch: Fetch, base: str) -> dict[str, str]: + refs = fetch(f"{base}/git/matching-refs/tags/") or [] + assert isinstance(refs, list) + return {r["ref"].removeprefix("refs/tags/"): "" for r in refs} + + +def resolve_tag(fetch: Fetch, base: str, tag: str) -> str: + """The commit a tag points at, peeling an annotated tag.""" + ref = fetch(f"{base}/git/ref/tags/{urllib.parse.quote(tag)}") + if not isinstance(ref, dict): + return "" + obj = ref["object"] + while obj["type"] == "tag": + tag_obj = fetch(f"{base}/git/tags/{obj['sha']}") + assert isinstance(tag_obj, dict) + obj = tag_obj["object"] + return str(obj["sha"]) + + +def read_marketplaces(fetch: Fetch, base: str) -> dict[str, set[str]]: + out: dict[str, set[str]] = {} + for market, path in MARKETPLACES.items(): + got = fetch(f"{base}/contents/{path}") + if not isinstance(got, dict) or "content" not in got: + out[market] = set() + continue + data = json.loads(base64.b64decode(got["content"])) + out[market] = {p["name"] for p in data.get("plugins", [])} + return out + + +# -------------------------------------------------------------------------- +# Comparing +# -------------------------------------------------------------------------- + + +def _version(text: str) -> tuple[int, ...]: + return tuple(int(p) for p in text.split(".")) + + +def tags_for(skill: str, tags: dict[str, str]) -> list[tuple[tuple[int, ...], str]]: + """``-`` tags for one skill, oldest first.""" + found = [] + for tag in tags: + m = _VERSIONED_REF_RE.match(tag) + if m and m.group("name") == skill: + found.append((_version(m.group("version")), tag)) + return sorted(found) + + +def check_pin(desc: Descriptor, upstream: Upstream, pin_commit: str, pinned_dirs: set[str]) -> list[str]: + """Findings for one descriptor's pin: the tag, its layout, newer tags.""" + findings: list[str] = [] + if desc.method == "git-tag" and desc.commit and pin_commit and pin_commit != desc.commit: + findings.append( + f"**Pinned tag moved.** `{desc.id}`: `{desc.ref}` now resolves to `{pin_commit[:12]}`, " + f"not the recorded `{desc.commit[:12]}`. Find out why before re-pinning." + ) + elif desc.method == "git-tag" and not pin_commit: + findings.append(f"**Pinned tag missing.** `{desc.id}`: `{desc.ref}` no longer exists in the source.") + for skill in desc.provides: + if skill not in pinned_dirs: + findings.append( + f"`{skill}` is in `{desc.id}`'s `provides` but is not a " + f"`{desc.skills_root}/{skill}/SKILL.md` directory at `{desc.ref}`, " + f"so `/magpie-setup:setup skill-sources` cannot wire it." + ) + pin_name = _VERSIONED_REF_RE.match(desc.ref) + if pin_name: + skill_tags = tags_for(pin_name.group("name"), upstream.tags) + if skill_tags and skill_tags[-1][0] > _version(pin_name.group("version")): + findings.append(f"**Newer tag** `{skill_tags[-1][1]}` for `{desc.id}` (pinned to `{desc.ref}`).") + return findings + + +def compare_catalogue(rows: list[CatalogueRow], upstream: Upstream, provided: set[str]) -> list[str]: + """Findings for the catalogue against the source's default branch.""" + findings: list[str] = [] + listed = {r.skill: r for r in rows} + for skill in sorted(set(upstream.layout) | set(listed)): + row, kind = listed.get(skill), upstream.layout.get(skill) + if row is None: + findings.append(f"**New skill** `{skill}` ({kind}) is not in the catalogue.") + continue + if kind is None: + findings.append(f"`{skill}` is in the catalogue but the source no longer ships it.") + continue + if skill in provided: + expected = "pinned" + elif kind == "directory": + expected = "untagged" + else: + expected = kind + if row.status != expected: + findings.append(f"`{skill}` is catalogued `{row.status}`, but the source makes it `{expected}`.") + actual = {market for market, names in upstream.marketplaces.items() if skill in names} + if actual != row.marketplaces: + shown = ", ".join(sorted(actual)) or "none" + findings.append( + f"`{skill}` installs from these marketplaces: {shown}; the catalogue says " + f"{', '.join(sorted(row.marketplaces)) or 'none'}." + ) + skill_tags = tags_for(skill, upstream.tags) + if skill_tags and skill not in provided and kind != "draft": + findings.append( + f"**Tagged** `{skill}` is now tagged (`{skill_tags[-1][1]}`) and could be pinned." + ) + return findings + + +def check_repo(fetch: Fetch, descs: list[Descriptor], rows: list[CatalogueRow] | None) -> list[str]: + """Check every descriptor that points at one repository, and its catalogue. + + A source that versions its skills independently is curated as one + descriptor per skill, each pinned to that skill's own tag; they share the + repository, so they share one catalogue.""" + m = _GITHUB_URL_RE.match(descs[0].url) + if not m: + return [f"Skipped: `{descs[0].url}` is not a GitHub repository; check it by hand."] + base = f"/repos/{m.group('owner')}/{m.group('repo')}" + upstream = Upstream( + layout=read_layout(fetch, base, descs[0].skills_root), + tags=read_tags(fetch, base), + marketplaces=read_marketplaces(fetch, base), + ) + findings: list[str] = [] + provided: set[str] = set() + for desc in descs: + pin_commit = resolve_tag(fetch, base, desc.ref) if desc.method == "git-tag" else "" + pinned_dirs = set() + for skill in desc.provides: + files = {e["name"] for e in _listing(fetch, base, f"{desc.skills_root}/{skill}", desc.ref)} + if "SKILL.md" in files: + pinned_dirs.add(skill) + provided.update(desc.provides) + findings += check_pin(desc, upstream, pin_commit, pinned_dirs) + if rows is None: + findings.append( + f"No catalogue for `{descs[0].url}`: add one so new skills in the source get noticed." + ) + else: + findings += compare_catalogue(rows, upstream, provided) + return findings + + +def run(root: Path, fetch: Fetch) -> tuple[int, str]: + sections: list[str] = [] + drift = False + for path in sorted((root / "organizations").glob("*/skill-sources.md")): + if path.parent.name.startswith("_"): + continue + rel = path.relative_to(root) + text = path.read_text(encoding="utf-8") + descriptors = {d.id: d for d in parse_descriptors(text)} + catalogues = parse_catalogues(text) + unknown = set(catalogues) - set(descriptors) + if unknown: + raise CheckError(f"{rel}: catalogue for unknown source(s) {sorted(unknown)}") + by_repo: dict[str, list[Descriptor]] = {} + for desc in descriptors.values(): + by_repo.setdefault(desc.url.rstrip("/").removesuffix(".git"), []).append(desc) + for descs in by_repo.values(): + ids = {d.id for d in descs} + named = [cid for cid in catalogues if cid in ids] + if len(named) > 1: + raise CheckError(f"{rel}: more than one catalogue for {descs[0].url}: {named}") + rows = catalogues[named[0]] if named else None + findings = check_repo(fetch, descs, rows) + title = f"### `{descs[0].url}` ({rel})" + if findings: + drift = drift or any(not f.startswith("Skipped:") for f in findings) + sections.append(title + "\n\n" + "\n".join(f"- {f}" for f in findings)) + else: + sections.append(title + "\n\nIn sync.") + report = "## Skill-source drift\n\n" + ("\n\n".join(sections) if sections else "No curated sources.") + return (1 if drift else 0), report + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--root", type=Path, default=ROOT, help="repository root") + args = parser.parse_args(argv) + try: + status, report = run(args.root, github_fetch) + except CheckError as exc: + print(f"check-skill-source-drift: {exc}", file=sys.stderr) + return 2 + sys.stdout.write(report) + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/dev/tests/test_check_skill_source_drift.py b/tools/dev/tests/test_check_skill_source_drift.py new file mode 100644 index 000000000..2392d3e64 --- /dev/null +++ b/tools/dev/tests/test_check_skill_source_drift.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for ``check-skill-source-drift.py``. + +The GitHub API is replaced by a dict of canned responses, so nothing here +touches the network. What matters: an in-sync catalogue passes, and each kind +of upstream change (a new skill, a removed one, a re-layout, a first tag, a +newer tag, a moved tag, a marketplace change) is reported as drift. +""" + +from __future__ import annotations + +import base64 +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[1] / "check-skill-source-drift.py" + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_skill_source_drift", _SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # dataclasses resolve their module through sys.modules. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +mod = _load() + +PIN = "a" * 40 +BASE = "/repos/apache/incubator" + +ORG_FILE = """\ +# Curated + + +| Skill | What it does | Install through Magpie | Marketplaces | +|---|---|---|---| +| `releasecheck` | Checks an RC. | `pinned` | `claude-code`, `codex` | +| `maturity` | Evidence pack. | `untagged` | — | +| `graduation` | Readiness. | `single-file` | — | +| `archaeology` | Draft. | `draft` | — | + + +```yaml +- id: apache-incubator + organization: ASF + name: "Apache Incubator skills" + method: git-tag + url: https://github.com/apache/incubator + ref: releasecheck-0.5.1 + commit: {pin} + layout: + skills_root: tools/skills + provides: + - skill: releasecheck +``` +""" + + +def _entries(*items: tuple[str, str]) -> list[dict[str, str]]: + return [{"name": n, "type": t} for n, t in items] + + +def _catalog(data: object) -> dict[str, str]: + return {"content": base64.b64encode(json.dumps(data).encode()).decode()} + + +def _responses() -> dict[str, object]: + market = {"plugins": [{"name": "releasecheck"}]} + return { + f"{BASE}/contents/tools/skills": _entries( + ("README.md", "file"), + ("graduation.md", "file"), + ("releasecheck", "dir"), + ("maturity", "dir"), + ("draft", "dir"), + ("evals", "dir"), + ), + f"{BASE}/contents/tools/skills/releasecheck": _entries(("SKILL.md", "file")), + f"{BASE}/contents/tools/skills/maturity": _entries(("SKILL.md", "file"), ("references", "dir")), + f"{BASE}/contents/tools/skills/draft": _entries(("archaeology.md", "file")), + f"{BASE}/contents/tools/skills/evals": _entries(("run-evals.sh", "file")), + f"{BASE}/contents/tools/skills/releasecheck?ref=releasecheck-0.5.1": _entries(("SKILL.md", "file")), + f"{BASE}/git/matching-refs/tags/": [ + {"ref": "refs/tags/releasecheck-0.5"}, + {"ref": "refs/tags/releasecheck-0.5.1"}, + {"ref": "refs/tags/releases/tools/1.0.0"}, + ], + f"{BASE}/git/ref/tags/releasecheck-0.5.1": {"object": {"type": "tag", "sha": "t1"}}, + f"{BASE}/git/tags/t1": {"object": {"type": "commit", "sha": PIN}}, + f"{BASE}/contents/.claude-plugin/marketplace.json": _catalog(market), + f"{BASE}/contents/.agents/plugins/marketplace.json": _catalog(market), + } + + +def _run(tmp_path: Path, responses: dict[str, object], org_file: str = ORG_FILE) -> tuple[int, str]: + org = tmp_path / "organizations" / "ASF" + org.mkdir(parents=True) + (org / "skill-sources.md").write_text(org_file.replace("{pin}", PIN), encoding="utf-8") + return mod.run(tmp_path, responses.get) + + +def test_in_sync(tmp_path: Path) -> None: + status, report = _run(tmp_path, _responses()) + assert status == 0, report + assert "In sync." in report + + +def test_new_skill_upstream(tmp_path: Path) -> None: + responses = _responses() + listing = responses[f"{BASE}/contents/tools/skills"] + assert isinstance(listing, list) + listing.append({"name": "board-report.md", "type": "file"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**New skill** `board-report` (single-file)" in report + + +def test_removed_skill(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/contents/tools/skills/draft"] = [] + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`archaeology` is in the catalogue but the source no longer ships it" in report + + +def test_single_file_became_directory(tmp_path: Path) -> None: + responses = _responses() + listing = responses[f"{BASE}/contents/tools/skills"] + assert isinstance(listing, list) + listing.remove({"name": "graduation.md", "type": "file"}) + listing.append({"name": "graduation", "type": "dir"}) + responses[f"{BASE}/contents/tools/skills/graduation"] = _entries(("SKILL.md", "file")) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`graduation` is catalogued `single-file`, but the source makes it `untagged`" in report + + +def test_first_tag_for_unpinned_skill(tmp_path: Path) -> None: + responses = _responses() + tags = responses[f"{BASE}/git/matching-refs/tags/"] + assert isinstance(tags, list) + tags.append({"ref": "refs/tags/maturity-0.1"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Tagged** `maturity` is now tagged (`maturity-0.1`)" in report + + +def test_newer_tag_for_pinned_skill(tmp_path: Path) -> None: + responses = _responses() + tags = responses[f"{BASE}/git/matching-refs/tags/"] + assert isinstance(tags, list) + tags.append({"ref": "refs/tags/releasecheck-0.10"}) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Newer tag** `releasecheck-0.10`" in report + + +def test_moved_tag(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/git/tags/t1"] = {"object": {"type": "commit", "sha": "b" * 40}} + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Pinned tag moved.**" in report + + +def test_missing_tag(tmp_path: Path) -> None: + responses = _responses() + del responses[f"{BASE}/git/ref/tags/releasecheck-0.5.1"] + status, report = _run(tmp_path, responses) + assert status == 1 + assert "**Pinned tag missing.**" in report + + +def test_marketplace_change(tmp_path: Path) -> None: + responses = _responses() + market = {"plugins": [{"name": "releasecheck"}, {"name": "maturity"}]} + responses[f"{BASE}/contents/marketplace.json"] = _catalog(market) + status, report = _run(tmp_path, responses) + assert status == 1 + assert "`maturity` installs from these marketplaces: copilot; the catalogue says none" in report + + +def test_unknown_marketplace_is_an_error(tmp_path: Path) -> None: + org_file = ORG_FILE.replace("`claude-code`, `codex` |", "`claude-code`, `nowhere` |") + with pytest.raises(mod.CheckError, match="unknown marketplace"): + _run(tmp_path, _responses(), org_file) + + +def test_provided_skill_not_a_directory_at_pin(tmp_path: Path) -> None: + responses = _responses() + responses[f"{BASE}/contents/tools/skills/releasecheck?ref=releasecheck-0.5.1"] = None + status, report = _run(tmp_path, responses) + assert status == 1 + assert "is in `apache-incubator`'s `provides` but is not a" in report + + +def test_catalogue_without_descriptor_is_an_error(tmp_path: Path) -> None: + org_file = ORG_FILE.replace("CATALOGUE: apache-incubator", "CATALOGUE: elsewhere") + with pytest.raises(mod.CheckError, match="unknown source"): + _run(tmp_path, _responses(), org_file) + + +def test_descriptor_without_catalogue_is_drift(tmp_path: Path) -> None: + start = ORG_FILE.index("", ORG_FILE.index("