From 3361cf6c5423f78b987d9ff3d8bf80f194a49bed Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Wed, 21 Oct 2026 18:57:28 -0400 Subject: [PATCH] Use componentized/build Signed-off-by: Scott Andrews --- .github/workflows/bump-version.yaml | 85 +++------- .github/workflows/ci.yaml | 188 ++++++++++++++++++++- .github/workflows/release.yaml | 56 ++++++ .github/workflows/update-build.yaml | 105 ++++++++++++ Makefile | 33 +--- build/MANIFEST | 20 +++ build/README.md | 10 ++ build/VERSION | 1 + build/build.mk | 81 +++++++++ build/components.mk | 127 ++++++++++++++ build/local.jq | 21 +++ {make => build}/pinned.jq | 2 +- build/publish.mk | 19 +++ {scripts => build/scripts}/bump-version.sh | 67 ++++++-- build/scripts/check-build.sh | 65 +++++++ build/scripts/create-commit.sh | 72 ++++++++ {scripts => build/scripts}/publish.sh | 65 ++++--- build/scripts/update-build.sh | 157 +++++++++++++++++ build/scripts/wac-deps.sh | 46 +++++ build/sync.mk | 10 ++ build/tool-versions | 6 + build/tools.mk | 117 +++++++++++++ build/wit.mk | 41 +++++ make/components.mk | 76 --------- make/publish.mk | 15 -- make/tools.mk | 53 ------ make/wit.mk | 28 --- scripts/init-devcontainer.sh | 12 -- 28 files changed, 1263 insertions(+), 315 deletions(-) create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/update-build.yaml create mode 100644 build/MANIFEST create mode 100644 build/README.md create mode 100644 build/VERSION create mode 100644 build/build.mk create mode 100644 build/components.mk create mode 100644 build/local.jq rename {make => build}/pinned.jq (91%) create mode 100644 build/publish.mk rename {scripts => build/scripts}/bump-version.sh (50%) create mode 100755 build/scripts/check-build.sh create mode 100755 build/scripts/create-commit.sh rename {scripts => build/scripts}/publish.sh (59%) create mode 100755 build/scripts/update-build.sh create mode 100755 build/scripts/wac-deps.sh create mode 100644 build/sync.mk create mode 100644 build/tool-versions create mode 100644 build/tools.mk create mode 100644 build/wit.mk delete mode 100644 make/components.mk delete mode 100644 make/publish.mk delete mode 100644 make/tools.mk delete mode 100644 make/wit.mk delete mode 100755 scripts/init-devcontainer.sh diff --git a/.github/workflows/bump-version.yaml b/.github/workflows/bump-version.yaml index b6115ba..0c0bda9 100644 --- a/.github/workflows/bump-version.yaml +++ b/.github/workflows/bump-version.yaml @@ -4,10 +4,12 @@ on: workflow_dispatch: inputs: version: - description: The new version of the interface package, e.g. 0.2.0 or 0.3.0-dev + description: The new version of the interface package and crates, e.g. 0.1.0 or 0.2.0-dev required: true type: string - default: "0.2.1-dev" # the current version, kept current by scripts/bump-version.sh + # the repository's own version, carried forward by build/scripts/update-build.sh, and ignored + # by build/scripts/check-build.sh + default: "0.2.1-dev" # the current version, kept current by build/scripts/bump-version.sh jobs: # bumps the version with read only access, the changes are handed to the push job as a patch so @@ -16,26 +18,35 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + outputs: + current: ${{ steps.version.outputs.current }} + version: ${{ steps.version.outputs.version }} steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false - name: Check version + id: version # a semver version without a leading `v`, the tag adds it. Build metadata is used to tag the # debug builds of components, e.g. 0.1.0+debug, and isn't allowed in the version itself. run: | + # the checkout is before the bump, the wit package's version is the current version + current=$( sed -n "s/^package ${GITHUB_REPOSITORY/\//:}@\(.*\);$/\1/p" wit/worlds.wit ) identifier='(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)' semver="^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-${identifier}(\.${identifier})*)?$" if ! [[ "${VERSION}" =~ ${semver} ]] ; then - echo "::error::the version '${VERSION}' is not a valid semver version, e.g. 0.2.0 or 0.3.0-dev" + echo "::error::the version '${VERSION}' is not a valid semver version, e.g. 0.1.0 or 0.2.0-dev" exit 1 fi if [[ "${VERSION}" == *+* ]] ; then echo "::error::the version '${VERSION}' must not contain build metadata" exit 1 fi + echo "bumping ${current} to ${VERSION}" + echo "current=${current}" >> "${GITHUB_OUTPUT}" + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" env: VERSION: ${{ inputs.version }} - - uses: actions/checkout@v7 - with: - persist-credentials: false - uses: actions-rust-lang/setup-rust-toolchain@v2 - name: Install cargo binstall uses: cargo-bins/cargo-binstall@main @@ -45,9 +56,9 @@ jobs: make -s tools-path >> "${GITHUB_PATH}" - name: Bump version # also fetches the wit dependencies for the new version, and builds and tests the components - run: scripts/bump-version.sh "${VERSION}" + run: build/scripts/bump-version.sh "${VERSION}" env: - VERSION: ${{ inputs.version }} + VERSION: ${{ steps.version.outputs.version }} - name: Collect changes run: | git add --all @@ -75,7 +86,8 @@ jobs: permissions: contents: read env: - VERSION: ${{ inputs.version }} + CURRENT_VERSION: ${{ needs.bump.outputs.current }} + VERSION: ${{ needs.bump.outputs.version }} steps: - name: Check custodian app credentials run: | @@ -95,7 +107,7 @@ jobs: # only this repository, with only the permissions the bump needs repositories: ${{ github.event.repository.name }} permission-contents: write - # the bump changes the default version in this workflow + # the bump may change a workflow file, e.g. a reference to the package permission-workflows: write # a release runs this workflow again, to bump to the next dev version permission-actions: write @@ -107,14 +119,8 @@ jobs: with: name: bump-version.patch path: ${{ runner.temp }} - - name: Read current version - # the checkout is before the bump, the wit package's version is the current version - run: | - current=$( sed -n 's/^package componentized:cli@\(.*\);$/\1/p' wit/worlds.wit ) - echo "CURRENT_VERSION=${current}" >> "${GITHUB_ENV}" - name: Commit changes - # the commit is created with the REST API, as the app's token can't push. The patch is applied - # locally only to find the changed files and their modes. + # the commit is created with the REST API, as the app's token can't push env: GH_TOKEN: ${{ steps.app-token.outputs.token }} APP_SLUG: ${{ steps.app-token.outputs.app-slug }} @@ -122,48 +128,7 @@ jobs: api="repos/${GITHUB_REPOSITORY}" base=$( git rev-parse HEAD ) git apply --index "${RUNNER_TEMP}/bump-version.patch" - - # a blob for each changed file, or a null sha for a deleted file - entries="${RUNNER_TEMP}/tree-entries.json" - echo '[]' > "${entries}" - git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do - if [ "${status}" = "D" ] ; then - entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' ) - else - mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 ) - sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha ) - entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' ) - fi - jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}" - echo "${status} ${path}" - done - tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha ) - - # authored, committed and signed off (DCO) by the custodian app's bot. GitHub only signs - # commits it attributes entirely to the app, so the author and committer are left for GitHub - # to fill in. The user who triggered the workflow is credited as a co-author, with their - # GitHub noreply email so their email address isn't exposed. A run started by the app after a - # release has no one else to credit. - bot="${APP_SLUG}[bot]" - bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com" - message=$( printf 'Bump version from %s to %s\n\nSigned-off-by: %s <%s>' "${CURRENT_VERSION}" "${VERSION}" "${bot}" "${bot_email}" ) - if [ "${GITHUB_ACTOR}" != "${bot}" ] ; then - name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' ) - name="${name:-${GITHUB_ACTOR}}" - email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - message=$( printf '%s\nCo-authored-by: %s <%s>' "${message}" "${name}" "${email}" ) - fi - read -r commit verified < <( jq -n \ - --arg message "${message}" \ - --arg tree "${tree}" --arg parent "${base}" \ - '{message: $message, tree: $tree, parents: [$parent]}' \ - | gh api --method POST "${api}/git/commits" --input - --jq '"\(.sha) \(.verification.verified)"' ) - echo "created commit ${commit}" - # the branch requires signed commits, fail before pushing rather than after - if [ "${verified}" != "true" ] ; then - echo "::error::the commit '${commit}' was not created, or was not signed by GitHub" - exit 1 - fi + commit=$( build/scripts/create-commit.sh "${base}" "Bump version from ${CURRENT_VERSION} to ${VERSION}" ) # a release, rather than a pre-release, is tagged, e.g. v0.1.0. Tags are immutable, check the # tag is free before pushing the branch, so a release isn't left without its tag @@ -193,7 +158,7 @@ jobs: - name: Bump to the next dev version # after a release, the branch moves on to a pre-release of the next patch version, e.g. 0.3.0 # is followed by 0.3.1-dev. Run with the app's token, the GITHUB_TOKEN can't start workflows. - if: ${{ !contains(inputs.version, '-') }} + if: ${{ !contains(needs.bump.outputs.version, '-') }} env: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 64d6a83..1067a4f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,21 @@ on: pull_request: {} jobs: + # the settings of the repository's Makefile the other jobs need, e.g. whether it builds a cli + settings: + runs-on: ubuntu-latest + outputs: + cli: ${{ steps.settings.outputs.cli }} + steps: + - uses: actions/checkout@v7 + - name: Check build + # the files synced from componentized/build are unchanged, see build/README.md + run: build/scripts/check-build.sh + - uses: actions-rust-lang/setup-rust-toolchain@v2 + - name: Read settings + id: settings + run: echo "cli=$( make -s print-CLI_BIN )" >> "${GITHUB_OUTPUT}" + build: runs-on: ubuntu-latest steps: @@ -22,10 +37,31 @@ jobs: run: | make tools make -s tools-path >> "${GITHUB_PATH}" - - name: Sync wit - run: make wit - - name: Check for drift in generated wit + - name: Generate + # the wit dependencies, and anything else the repository generates, see `make generate` + run: make generate + - name: Check for drift in generated files run: git diff --exit-code . + - name: Check crate version matches the interface version + # the crates inherit the workspace version, bumped with the interface by + # build/scripts/bump-version.sh. A repository without a workspace version has nothing to check. + run: | + interface_version=$( sed -n "s/^package ${GITHUB_REPOSITORY/\//:}@\(.*\);$/\1/p" wit/worlds.wit ) + workspace_version=$( sed -n '/^\[workspace.package\]/,/^\[/s/^version = "\(.*\)"$/\1/p' Cargo.toml ) + if [ -n "${workspace_version}" ] && [ "${workspace_version}" != "${interface_version}" ] ; then + echo "::error::the workspace version ${workspace_version} in Cargo.toml does not match the interface version ${interface_version}, see build/scripts/bump-version.sh" + exit 1 + fi + - name: Check crates can be published + # the release workflow publishes the crates when a release is published, too late to find a + # crate can't be published. A cli is verified against the library being published with it. + run: | + read -ra crates <<< "$( make -s print-PUBLISH_CRATES )" + if [ "${#crates[@]}" -eq 0 ] ; then + echo "no crates to publish, see PUBLISH_CRATES" + exit 0 + fi + cargo publish --dry-run --locked "${crates[@]/#/--package=}" - name: Build components run: make components - name: Collect components.tar @@ -39,13 +75,16 @@ jobs: retention-days: 7 - name: Test run: make test + - name: Integration test + # the tests that need more than cargo, e.g. a server in a container, see `make test-integration` + run: make test-integration - name: Capture WIT working-directory: ./target/components run: | dump_wit() { echo "::group::$(basename "$1") ($(du -h "$1" | awk '{print $1}' ))" wasm-tools component wit "$1" - echo "::endgroup::" + echo "::endgroup::" } # print interface.wasm first @@ -56,16 +95,130 @@ jobs: dump_wit "${component}" done - publish: - if: github.event_name == 'push' && ( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' ) + # the cli built from the repository, see CLI_BIN, packaged for each platform as cargo binstall + # expects, e.g. constants-aarch64-apple-darwin.tar.gz + cli: needs: + - settings + if: needs.settings.outputs.cli != '' + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-26.04 + target: x86_64-unknown-linux-musl + - os: ubuntu-26.04-arm + target: aarch64-unknown-linux-musl + - os: macos-26 + target: aarch64-apple-darwin + - os: macos-26-intel + target: x86_64-apple-darwin + - os: windows-2025 + target: x86_64-pc-windows-msvc + - os: windows-11-arm + target: aarch64-pc-windows-msvc + # each runner builds for its own platform, linux builds are static with musl + runs-on: ${{ matrix.os }} + # bash on windows too, where pwsh is the default, the steps read CLI_BIN from the environment. The + # windows packaging steps are pwsh, reading it as `${env:CLI_BIN}`. + defaults: + run: + shell: bash + env: + CLI_BIN: ${{ needs.settings.outputs.cli }} + steps: + - uses: actions/checkout@v7 + - uses: actions-rust-lang/setup-rust-toolchain@v2 + with: + target: ${{ matrix.target }} + - name: Build cli + run: cargo build --release --locked --bin "${CLI_BIN}" --target ${{ matrix.target }} + - name: Package cli + if: runner.os != 'Windows' + run: | + mkdir -p dist + tar -czvf "dist/${CLI_BIN}-${{ matrix.target }}.tar.gz" -C target/${{ matrix.target }}/release "${CLI_BIN}" -C "${PWD}" LICENSE NOTICE README.md + - name: Package cli + if: runner.os == 'Windows' + shell: pwsh + run: | + New-Item -ItemType Directory -Force dist + Compress-Archive -Path "target/${{ matrix.target }}/release/${env:CLI_BIN}.exe", LICENSE, NOTICE, README.md -DestinationPath "dist/${env:CLI_BIN}-${{ matrix.target }}.zip" + # the smoke test runs the cli from the package, as a user would + - name: Unpack cli + if: runner.os != 'Windows' + run: | + mkdir -p smoke + tar -xzvf "dist/${CLI_BIN}-${{ matrix.target }}.tar.gz" -C smoke + - name: Unpack cli + if: runner.os == 'Windows' + shell: pwsh + run: Expand-Archive -Path "dist/${env:CLI_BIN}-${{ matrix.target }}.zip" -DestinationPath smoke + - name: Install wasmtime + uses: bytecodealliance/actions/wasmtime/setup@v1 + - name: Smoke test cli + # the repository's scripts/smoke-test-cli.sh runs the unpacked cli, given its path and a + # directory for its files + run: | + if [ ! -f scripts/smoke-test-cli.sh ] ; then + echo "::warning::no scripts/smoke-test-cli.sh to smoke test ${CLI_BIN}" + exit 0 + fi + bash scripts/smoke-test-cli.sh "./smoke/${CLI_BIN}" smoke + - name: Upload cli + uses: actions/upload-artifact@v7 + with: + name: cli-${{ matrix.target }} + path: dist/* + if-no-files-found: error + retention-days: 7 + + # a single check for branch protection to require, rather than each job and matrix entry. It always + # runs, a skipped required check counts as passing, and fails when any job it needs did not succeed. + # The cli is skipped for a repository without one. + ci: + if: always() + needs: + - settings - build + - cli + runs-on: ubuntu-latest + steps: + - name: Check jobs succeeded + env: + RESULTS: ${{ toJSON(needs.*.result) }} + CLI: ${{ needs.settings.outputs.cli }} + CLI_RESULT: ${{ needs.cli.result }} + run: | + echo "results: ${RESULTS}" + if [ -z "${CLI}" ] && [ "${CLI_RESULT}" = "skipped" ] ; then + RESULTS=$( jq -c 'map(select(. != "skipped"))' <<< "${RESULTS}" ) + fi + if jq -e 'any(. != "success")' <<< "${RESULTS}" > /dev/null ; then + echo "::error::a required job did not succeed" + exit 1 + fi + + publish: + # the ci job's result rather than the default success(), which is false when the cli is skipped, + # even though the ci job succeeded without it + if: ${{ !cancelled() && needs.ci.result == 'success' && github.event_name == 'push' && ( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' ) }} + needs: + - settings + - ci permissions: contents: write packages: write id-token: write runs-on: ubuntu-latest + env: + # the repository variable DRY_RUN=true goes through publishing without publishing anything: + # nothing is pushed or signed, and no release is drafted + DRY_RUN: ${{ vars.DRY_RUN == 'true' }} steps: + - name: Dry run + if: env.DRY_RUN == 'true' + run: echo "::notice::a dry run, the repository variable DRY_RUN is true, nothing is published" - uses: actions/checkout@v7 - uses: actions-rust-lang/setup-rust-toolchain@v2 - name: Install cargo binstall @@ -80,6 +233,13 @@ jobs: uses: actions/download-artifact@v8 with: name: components.tar + - name: Download cli + if: needs.settings.outputs.cli != '' + uses: actions/download-artifact@v8 + with: + pattern: cli-* + path: dist + merge-multiple: true - name: Extract components run: mkdir -p target/components && tar -xvf components.tar -C target/components - name: Get interface version @@ -89,7 +249,19 @@ jobs: if: startsWith(github.ref, 'refs/tags/') id: tag_version run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT + - name: Check tag matches the workspace version + # cargo binstall finds the cli archives at the release tagged with the cli crate's version, + # and publishing the release publishes the crates to crates.io at that version. A repository + # without a workspace version has nothing to check. + if: startsWith(github.ref, 'refs/tags/') + run: | + workspace_version=$( sed -n '/^\[workspace.package\]/,/^\[/s/^version = "\(.*\)"$/\1/p' Cargo.toml ) + if [ -n "${workspace_version}" ] && [ "v${workspace_version}" != "${{ steps.tag_version.outputs.VERSION }}" ] ; then + echo "::error::tag ${{ steps.tag_version.outputs.VERSION }} does not match the workspace version ${workspace_version} in Cargo.toml" + exit 1 + fi - name: Login to GitHub Container Registry + if: env.DRY_RUN != 'true' uses: docker/login-action@v4 with: registry: ghcr.io @@ -118,12 +290,14 @@ jobs: } > "${RUNNER_TEMP}/release-notes.md" cat "${RUNNER_TEMP}/release-notes.md" - name: Draft GitHub Release - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') && ( env.DRY_RUN != 'true' || github.repository == 'componentized/build' ) uses: softprops/action-gh-release@v3 with: draft: true body_path: ${{ runner.temp }}/release-notes.md + # the cli archives of a repository with a cli files: | components.tar + ${{ needs.settings.outputs.cli != '' && 'dist/*' || '' }} fail_on_unmatched_files: true token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..58b0c44 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,56 @@ +name: Release + +on: + release: + types: + - published + +jobs: + # the crates of a repository that publishes crates, see PUBLISH_CRATES + crates: + runs-on: ubuntu-latest + permissions: + contents: read + # crates.io trusted publishing exchanges a GitHub OIDC token for a short lived crates.io token + id-token: write + env: + # the repository variable DRY_RUN=true checks the crates with `cargo publish --dry-run` rather + # than publishing them + DRY_RUN: ${{ vars.DRY_RUN == 'true' }} + steps: + - uses: actions/checkout@v7 + - uses: actions-rust-lang/setup-rust-toolchain@v2 + - name: Read settings + id: settings + run: | + crates=$( make -s print-PUBLISH_CRATES ) + [ -z "${crates}" ] && echo "no crates to publish, see PUBLISH_CRATES" + echo "crates=${crates}" >> "${GITHUB_OUTPUT}" + - name: Check crate version matches the release + # the crates inherit the workspace version + if: steps.settings.outputs.crates != '' + run: | + tag="${{ github.event.release.tag_name }}" + workspace_version=$( sed -n '/^\[workspace.package\]/,/^\[/s/^version = "\(.*\)"$/\1/p' Cargo.toml ) + if [ "v${workspace_version}" != "${tag}" ] ; then + echo "::error::release ${tag} does not match the workspace version ${workspace_version} in Cargo.toml" + exit 1 + fi + - name: Authenticate with crates.io + if: steps.settings.outputs.crates != '' && env.DRY_RUN != 'true' + id: auth + uses: rust-lang/crates-io-auth-action@v1 + - name: Publish crates + # cargo publishes a library before the cli that depends on it + if: steps.settings.outputs.crates != '' + run: | + read -ra crates <<< "${CRATES}" + dry_run=() + if [ "${DRY_RUN}" = "true" ] ; then + echo "::notice::a dry run, the repository variable DRY_RUN is true, the crates are not published" + dry_run=(--dry-run) + fi + cargo publish --locked "${dry_run[@]}" "${crates[@]/#/--package=}" + env: + CRATES: ${{ steps.settings.outputs.crates }} + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/.github/workflows/update-build.yaml b/.github/workflows/update-build.yaml new file mode 100644 index 0000000..da3d047 --- /dev/null +++ b/.github/workflows/update-build.yaml @@ -0,0 +1,105 @@ +name: Update build + +on: + schedule: + # weekly, Monday morning + - cron: "17 6 * * 1" + workflow_dispatch: + inputs: + version: + description: The release of componentized/build to update to, e.g. v0.2.0. Empty updates to the latest release. + required: false + type: string + +jobs: + # opens a pull request updating the build synced from componentized/build to its latest release, like + # dependabot. The pull request is left for the repository to merge when it's ready. + update: + # componentized/build is the source of the build, it has no synced copy to update + if: github.repository != 'componentized/build' + runs-on: ubuntu-latest + # the branch and pull request are created with a token for the custodian GitHub App rather than the + # GITHUB_TOKEN, which can't change workflow files and doesn't trigger the CI workflow + permissions: + contents: read + steps: + - name: Check custodian app credentials + run: | + if [ -z "${CLIENT_ID}" ] || [ -z "${PRIVATE_KEY}" ] ; then + echo "::error::the CUSTODIAN_CLIENT_ID and CUSTODIAN_PRIVATE_KEY secrets must be available to this repository, the private key of the custodian GitHub App is needed to create a token" + exit 1 + fi + env: + CLIENT_ID: ${{ secrets.CUSTODIAN_CLIENT_ID }} + PRIVATE_KEY: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Find release + id: release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + run: | + if [ -z "${VERSION}" ] ; then + VERSION=$( gh api repos/componentized/build/releases/latest --jq .tag_name ) + fi + current=$( cat build/VERSION ) + echo "current=${current}" >> "${GITHUB_OUTPUT}" + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + if [ "${VERSION}" = "${current}" ] ; then + echo "the build is already componentized/build ${current}" + fi + - name: Create custodian app token + if: steps.release.outputs.version != steps.release.outputs.current + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + # only this repository, with only the permissions the update needs + repositories: ${{ github.event.repository.name }} + permission-contents: write + # the build includes workflow files + permission-workflows: write + permission-pull-requests: write + - name: Update build + if: steps.release.outputs.version != steps.release.outputs.current + run: build/scripts/update-build.sh "${VERSION}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.release.outputs.version }} + - name: Open pull request + if: steps.release.outputs.version != steps.release.outputs.current + # the commit is created with the REST API, as the app's token can't push + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + CURRENT_VERSION: ${{ steps.release.outputs.current }} + VERSION: ${{ steps.release.outputs.version }} + # a scheduled run has no one to credit as a co-author + GITHUB_ACTOR: ${{ github.event_name == 'workflow_dispatch' && github.actor || '' }} + run: | + api="repos/${GITHUB_REPOSITORY}" + branch="update-build/${VERSION}" + # a pull request for the release was already opened, maybe closed, it isn't opened again + if gh api "${api}/git/ref/heads/${branch}" --silent 2> /dev/null ; then + echo "the branch ${branch} already exists" + exit 0 + fi + + base=$( git rev-parse HEAD ) + git add --all + commit=$( build/scripts/create-commit.sh "${base}" "Update build from ${CURRENT_VERSION} to ${VERSION}" ) + gh api --method POST "${api}/git/refs" -f ref="refs/heads/${branch}" -f sha="${commit}" --silent + echo "pushed ${commit} to ${branch}" + + { + echo "Updates the build synced from [componentized/build](https://github.com/componentized/build) from ${CURRENT_VERSION} to ${VERSION}, see the [release notes](https://github.com/componentized/build/releases/tag/${VERSION})." + echo + echo '```' + git diff --cached --stat "${base}" + echo '```' + } > "${RUNNER_TEMP}/body.md" + gh pr create --repo "${GITHUB_REPOSITORY}" --base "${GITHUB_REF_NAME}" --head "${branch}" \ + --title "Update build to ${VERSION}" --body-file "${RUNNER_TEMP}/body.md" diff --git a/Makefile b/Makefile index 9e26af7..ba04349 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,4 @@ -SHELL := /bin/bash +# This repository builds with its own build, its components and cli exist to smoke test it, see +# README.md -export RUST_BACKTRACE ?= 1 -export WASMTIME_BACKTRACE_DETAILS ?= 1 -export WKG_CONFIG_FILE := $(abspath .config/wasm-pkg/config.toml) - -# `make` without a goal runs all, otherwise the first target of an included file would run -.DEFAULT_GOAL := all - -# tools first, the others use the variables it defines for each tool when their rules are read -include make/tools.mk -include make/wit.mk -include make/components.mk -include make/publish.mk - - -.PHONY: all -all: tools wit components test - -.PHONY: clean -clean: clean-components clean-wit - @: - -.PHONY: clean-all -clean-all: clean-components clean-tools clean-wit - cargo clean - - -.PHONY: test -test: components - cargo test --workspace +include build/build.mk diff --git a/build/MANIFEST b/build/MANIFEST new file mode 100644 index 0000000..c1816ec --- /dev/null +++ b/build/MANIFEST @@ -0,0 +1,20 @@ +0edf07b7ec073ce666d847f9c9976ad68822d80d7c1651739459ff27f8b3fa0f .github/workflows/bump-version.yaml +fb1340584ef4774b44a80acc8112ef7fa5243dc61d63f19a1bd09bd64c2e7a94 .github/workflows/ci.yaml +b09c55d4207eb188036d3a7ba118c68b2547e958362d898714408af3a1c4afa6 .github/workflows/release.yaml +0a5da65531ed134a43f0aba94aacccd84707d548c348eb31868365c8c400978a .github/workflows/update-build.yaml +f9b6ad1d205adf8198aecdf9130fe149a362cbe805c077f19c17dc5562b60405 build/README.md +79a4951068b8e2e41666b044db6911a5b851b4bad0b48eff66803e6356e3fc5d build/build.mk +bc6e831ba0d2363c43ac590fea4944bfd57f2767d6c9f89022972c54fab274df build/components.mk +acb1b32ba485536dc0b834cb351f41a6804339e4beab0b0391d149d069724a89 build/local.jq +7a92f0051f300b0e22f571e34d441c6cd93ccbfbecb3bf3cdbff7083f8124bdd build/pinned.jq +df556e7ce9919c72815db01ac06ef541ecb1b9f920facb51ccbb635d17ee656a build/publish.mk +2e98011760ad48ff0d706df309f95a067e0edc82c9ec55292cd2522d4e347c47 build/scripts/bump-version.sh +c651f8d58218a8ad6475857a70364e2df9e5607aa066800f19d0b65ec97a02da build/scripts/check-build.sh +45c5f4b5e7a4525c6da2780687b705536f2a57f40c18eda0ec84015c5c38df3e build/scripts/create-commit.sh +9cef942742a92e6c8845b72ca59ba9c13154415956c1be7dab72d61acacbfc2e build/scripts/publish.sh +d8d869c6df5dbaa0a11356e34e5e82b64587a47969b085f14b7037d96b2e108e build/scripts/update-build.sh +0477a5e4ee8fdb61e6cf67205c43aafe52be25501b8e9d41e72ce805b5b3c0df build/scripts/wac-deps.sh +44f65271d65b043e9affcbaddd9e958ecb33a60e945d7ce869e6408925620626 build/sync.mk +0a52ef86f584169f9f2e0fd5f67ac9add84b8186fbb21d197508373eaa3ed9bc build/tool-versions +e342fc2d63250dda36ee8f85b34cac4930921052414c97c1f1c46a667e9e3e06 build/tools.mk +7010449b71c86de391f25e32a8bf06419f1a277479108aa31250b8aead56e45d build/wit.mk diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..11a1d43 --- /dev/null +++ b/build/README.md @@ -0,0 +1,10 @@ +# Build + +The build shared by the componentized repositories, synced from [componentized/build](https://github.com/componentized/build) at the version in [`VERSION`](VERSION). + +Nothing in this directory, nor the workflows synced with it, is edited here: `make check-build`, run by the CI workflow, fails when a synced file differs from [`MANIFEST`](MANIFEST). Changes go to componentized/build, and come back with its next release: + +- `make update-build` updates to the latest release, `make update-build BUILD_VERSION=v0.2.0` to a given one. +- The `Update build` workflow opens a pull request each week when there is a newer release. + +The repository's own `Makefile` sets the settings of the build, includes [`build.mk`](build.mk), then adds its own targets. See the [componentized/build README](https://github.com/componentized/build#readme) for the settings and the targets a repository can extend. diff --git a/build/VERSION b/build/VERSION new file mode 100644 index 0000000..b82608c --- /dev/null +++ b/build/VERSION @@ -0,0 +1 @@ +v0.1.0 diff --git a/build/build.mk b/build/build.mk new file mode 100644 index 0000000..9d3e98e --- /dev/null +++ b/build/build.mk @@ -0,0 +1,81 @@ +# The build shared by the componentized repositories, synced from https://github.com/componentized/build +# at the version in build/VERSION. Nothing in build/ is edited here, see build/README.md. +# +# The repository's Makefile sets its settings, includes this file, then adds its own targets and +# prerequisites, e.g. +# +# COMPONENT_TARGET := wasm32-wasip3 +# include build/build.mk +# test-integration: test-valkey + +SHELL := /bin/bash + +export RUST_BACKTRACE ?= 1 +export WASMTIME_BACKTRACE_DETAILS ?= 1 +export WKG_CONFIG_FILE := $(abspath .config/wasm-pkg/config.toml) + +# `make` without a goal runs all, otherwise the first target of an included file would run +.DEFAULT_GOAL := all + +# the rust target components are built for, `wasm32-wasip3` builds components that run std threads on +# component model cooperative threads, see build/components.mk +COMPONENT_TARGET ?= wasm32-unknown-unknown + +# the cli built from this repository, e.g. `constants`, released with an archive for each platform by +# the CI workflow. Empty for a repository without a cli. +CLI_BIN ?= +# the crates published to crates.io when a release is published, e.g. `componentized-constants +# componentized-constants-cli`, checked by the CI workflow. Empty for a repository without crates. +PUBLISH_CRATES ?= + +# tools first, the others use the variables it defines for each tool when their rules are read +include build/tools.mk +include build/wit.mk +include build/components.mk +include build/publish.mk +include build/sync.mk + + +.PHONY: all ## Install the tools, generate, build the components and test them, `make` without a target +all: tools generate components test + +# the help of a target follows `## ` on its .PHONY line, e.g. `.PHONY: tools ## Install the tools`, or +# on the line of its rule, e.g. `run: cli.wasm ## Run the cli`. A variable's line, with an `=`, is not +# a target. +.PHONY: help ## Display this help screen +help: + @awk '/^[^#[:space:]][^=]*:[^=]*## / { \ + names = substr($$0, 1, index($$0, "## ") - 1); \ + if (sub(/^\.PHONY:/, "", names) == 0) sub(/:.*$$/, "", names); \ + text = substr($$0, index($$0, "## ") + 3); \ + count = split(names, list, /[[:space:]]+/); \ + for (i = 1; i <= count; i++) if (list[i] != "" && !(list[i] in seen)) { seen[list[i]] = 1; printf "\033[36m%-20s\033[0m %s\n", list[i], text } \ + }' $(MAKEFILE_LIST) | sort + +# the repository adds what else it generates and commits as prerequisites, e.g. `generate: config` +.PHONY: generate ## Generate the files committed with the sources, the CI workflow fails when they drift +generate: wit + @: + +.PHONY: clean ## Remove the built components and the fetched wit dependencies +clean: clean-components clean-wit + @: + +.PHONY: clean-all ## Remove everything built, the installed tools and cargo's target directory too +clean-all: clean-components clean-tools clean-wit + cargo clean + +.PHONY: test ## Build the components and run the tests of the workspace +test: components + cargo test --workspace + +# the repository adds the tests that need more than cargo as prerequisites, e.g. a server in a +# container, `test-integration: test-valkey` +.PHONY: test-integration ## Run the tests that need more than cargo, run by the CI workflow after `make test` +test-integration: + @: + +# prints the value of a variable, e.g. `make -s print-CLI_BIN`, for the workflows +print-%: + $(info $($*)) + @: diff --git a/build/components.mk b/build/components.mk new file mode 100644 index 0000000..cf276d7 --- /dev/null +++ b/build/components.mk @@ -0,0 +1,127 @@ +COMPONENTS_DIR := target/components + +# the WIT packages of components/wit/deps, encoded where `wac compose` looks for a package a +# composition names without instantiating it, e.g. `import stderr: wasi:cli/stderr@0.3.0;` +WAC_DEPS_DIR := target/wac-deps + +COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) + +# the cargo profiles components are built with, `component` and `component-debug` when Cargo.toml +# defines them, e.g. to build smaller components than the release profile a cli is built with +COMPONENT_PROFILE ?= $(if $(shell grep -s '^\[profile\.component\]' Cargo.toml),component,release) +COMPONENT_DEBUG_PROFILE ?= $(if $(shell grep -s '^\[profile\.component-debug\]' Cargo.toml),component-debug,dev) + +# the core module cargo built for a component with a profile, e.g. `$(call cargo_wasm,gate,dev)` for +# target/wasm32-unknown-unknown/debug/gate.wasm +cargo_wasm = target/$(COMPONENT_TARGET)/$(if $(filter dev,$(2)),debug,$(2))/$(subst -,_,$(1)).wasm + +# the files the components cargo builds need besides the sources, e.g. a component the library +# embeds, `CARGO_COMPONENT_DEPS := crates/componentized-static-config/constants-config.wasm` +CARGO_COMPONENT_DEPS ?= + +ifeq ($(COMPONENT_TARGET),wasm32-wasip3) +# wasm32-wasip3 runs std threads on component model cooperative threads, which needs a wasi-libc built +# for them from wasi-sdk 34+'s experimental sysroot, installed with the tools or from WASI_SDK_PATH, and +# a rust toolchain with cooperative thread support, 1.100+. `make components +# COMPONENT_TARGET=wasm32-unknown-unknown` builds without. +WASI_SYSROOT_LIB := $(WASI_SDK_PATH)/share/wasi-sysroot/experimental-coop-threads/lib/wasm32-wasip3 +# linked with wasm-component-ld, like the target's default, against the coop threads wasi-libc in +# place of the libc vendored with std. Linked by clang, a cdylib is a shared library, not a component. +export CARGO_TARGET_WASM32_WASIP3_LINKER := $(WASI_SDK_PATH)/bin/wasm-component-ld +export CARGO_TARGET_WASM32_WASIP3_RUSTFLAGS := -Clink-self-contained=n -Clink-arg=-L$(WASI_SYSROOT_LIB) -Clink-arg=$(WASI_SYSROOT_LIB)/crt1-reactor.o -Clink-arg=--export=cabi_realloc +# turn the core module cargo built into a component, wasm32-wasip3 builds a component +componentize = cp $(1) $(2) +else +componentize = $(WASM_TOOLS) component new $(1) -o $(2) +endif + +# the release build of a component is optimized again for size with wasm-opt when it is pinned in +# tools/Cargo.toml, `COMPONENT_OPTIMIZE=` builds without. Only a core module is optimized, +# wasm32-wasip3 builds a component. +COMPONENT_OPTIMIZE ?= $(if $(filter wasm-opt,$(TOOLS)),$(filter-out wasm32-wasip3,$(COMPONENT_TARGET))) + +# the wasm features rust enables for wasm32-unknown-unknown, `rustc --print cfg --target +# wasm32-unknown-unknown`. The stripped modules don't say which features they use, wasm-opt is +# limited to these so it doesn't introduce instructions rust wouldn't emit. +WASM_OPT_FEATURES ?= --enable-bulk-memory --enable-multivalue --enable-mutable-globals --enable-nontrapping-float-to-int --enable-reference-types --enable-sign-ext + +.PHONY: clean-components ## Remove the built components +clean-components: clean-wit + rm -rf ${COMPONENTS_DIR} ${WAC_DEPS_DIR} + +.PHONY: components ## Build the interface and each component into target/components, `make components/` builds one +components: ${COMPONENTS_DIR}/interface.wasm $(foreach component,$(COMPONENTS),${COMPONENTS_DIR}/$(component)/$(component).wasm ${COMPONENTS_DIR}/$(component)/$(component).debug.wasm) + +define BUILD_COMPONENT + +.PHONY: components/$1 +components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm + +ifneq ($(wildcard components/$1/wit/$1.constants.wit),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md $(call tool_prerequisite,componentized-constants-cli) + $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md $(call tool_prerequisite,componentized-constants-cli) + $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +else ifneq ($(wildcard components/$1/$1.properties),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md $(call tool_prerequisite,componentized-static-config-cli) + $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md $(call tool_prerequisite,componentized-static-config-cli) + $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +else ifneq ($(wildcard components/$1/$1.wac),) + +# the local packages the composition instantiates, e.g. `new local:latch-n2 { ... }` +WAC_DEPS_$1 := $$(shell grep -v '^\s*//' components/$1/$1.wac | grep -oE 'local:[a-z0-9-]+' | sed 's/^local://' | sort -u) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) ${COMPONENTS_DIR}/$1/README.md ${WAC_DEPS_DIR} | $(call tool,wac-cli) + $(WAC) compose --deps-dir ${WAC_DEPS_DIR} $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) -o ${COMPONENTS_DIR}/$1/$1.wasm components/$1/$1.wac + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md ${WAC_DEPS_DIR} | $(call tool,wac-cli) + $(WAC) compose --deps-dir ${WAC_DEPS_DIR} $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm components/$1/$1.wac + +else ifneq ($(wildcard components/$1/$1.wkg),) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) + $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) + $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | tail -1 2> /dev/null) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + +# cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources +else ifneq ($(wildcard components/$1/Cargo.toml),) + +${COMPONENTS_DIR}/$1/$1.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(COMPONENT_OPTIMIZE),$(call tool,wasm-opt)) $(WASI_SDK) + cargo build -p $1 --target $(COMPONENT_TARGET) --profile $(COMPONENT_PROFILE) +ifneq ($(COMPONENT_OPTIMIZE),) + $(WASM_OPT) -Oz $(WASM_OPT_FEATURES) $(call cargo_wasm,$1,$(COMPONENT_PROFILE)) -o $(call cargo_wasm,$1.opt,$(COMPONENT_PROFILE)) + $(call componentize,$(call cargo_wasm,$1.opt,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm) +else + $(call componentize,$(call cargo_wasm,$1,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm) +endif + +${COMPONENTS_DIR}/$1/$1.debug.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(WASI_SDK) + cargo build -p $1 --target $(COMPONENT_TARGET) --profile $(COMPONENT_DEBUG_PROFILE) + $(call componentize,$(call cargo_wasm,$1,$(COMPONENT_DEBUG_PROFILE)),${COMPONENTS_DIR}/$1/$1.debug.wasm) + +endif + +${COMPONENTS_DIR}/$1/README.md: components/$1/README.md + @mkdir -p ${COMPONENTS_DIR}/$1 + @cp components/$1/README.md ${COMPONENTS_DIR}/$1/README.md + +endef + +$(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component)))) + +${WAC_DEPS_DIR}: components/wit/deps build/scripts/wac-deps.sh | $(call tool,wasm-tools) + WASM_TOOLS=$(WASM_TOOLS) build/scripts/wac-deps.sh components/wit/deps ${WAC_DEPS_DIR} + +${COMPONENTS_DIR}/interface.wasm: wit/deps README.md | $(call tool,wkg) + @mkdir -p ${COMPONENTS_DIR} + $(call wkg,.,build -o $(abspath ${COMPONENTS_DIR})/interface.wasm) + @cp README.md ${COMPONENTS_DIR}/README.md diff --git a/build/local.jq b/build/local.jq new file mode 100644 index 0000000..59bcc52 --- /dev/null +++ b/build/local.jq @@ -0,0 +1,21 @@ +# generates target/tools/local.mk from `cargo metadata --no-deps` of the workspace, for the packages +# named in $tools, LOCAL_TOOLS, see build/tools.mk +.workspace_root as $root +| (.packages | map({key: .name, value: .}) | from_entries) as $packages +| def relative: if . == $root then "." else ltrimstr($root + "/") end; + def dir: sub("/[^/]*$"; ""); + # the workspace packages a package depends on by path, but for tests, cargo forbids cycles + def path_deps: .dependencies[] | select(.path != null and .kind != "dev") | $packages[.name] | select(. != null); + # the manifest and the sources of a package's targets, but its tests, benches and examples. A + # target in the package's own directory, e.g. build.rs, is a file, others are their directory, + # e.g. src, a root package's directory holds the whole repository. + def sources: + (.manifest_path | dir) as $dir + | (.manifest_path | relative), + (.targets[] | select(any(.kind[]; IN("lib", "rlib", "cdylib", "staticlib", "dylib", "bin", "proc-macro", "custom-build"))) + | .src_path | if dir == $dir then relative else dir | relative end); + ($tools | split(" ") | map(select(. != "")))[] as $name + | ($packages[$name] // error("\($name) in LOCAL_TOOLS is not a package of the workspace")) as $package + | "LOCAL_TOOL_PATH_\($name) := \($package.manifest_path | dir | relative)", + "LOCAL_TOOL_SOURCES_\($name) := \([$package | recurse(path_deps) | sources] | unique | join(" "))", + ($package.targets[] | select(.kind | index("bin")) | .name | "\(ascii_upcase | gsub("-"; "_")) := $(TOOLS_BIN)/\(.)") diff --git a/make/pinned.jq b/build/pinned.jq similarity index 91% rename from make/pinned.jq rename to build/pinned.jq index 78fc559..f7b4b7b 100644 --- a/make/pinned.jq +++ b/build/pinned.jq @@ -1,4 +1,4 @@ -# generates target/tools/pinned.mk from `cargo metadata --no-deps` of tools/Cargo.toml, see make/tools.mk +# generates target/tools/pinned.mk from `cargo metadata --no-deps` of tools/Cargo.toml, see build/tools.mk .packages[0] as $tools | ($tools.dependencies[].name | select($tools.metadata.bins[.] == null) | error("\(.) is missing from [package.metadata.bins] in tools/Cargo.toml")), "TOOLS := \([$tools.dependencies[].name] | join(" "))", diff --git a/build/publish.mk b/build/publish.mk new file mode 100644 index 0000000..a675543 --- /dev/null +++ b/build/publish.mk @@ -0,0 +1,19 @@ +# sign published components with cosign, `SIGN=false` to push without signing, e.g. to a local registry +SIGN ?= true +# show what would be published without pushing or signing anything, `DRY_RUN=true` +DRY_RUN ?= false +# append each published file, its size in bytes and its image to this file, e.g. `gate.wasm 1234567 ghcr.io/componentized/http/gate:0.1.0@sha256:...` +PUBLISH_LOG ?= + +# the files that can be published, e.g. gate.wasm, published from target/components/gate/gate.wasm. +# Components are not published when they are pulled from elsewhere, `dep-`, built here for use by +# other components, `internal-`, or tests, `test-`. +PUBLISH_FILES := interface.wasm $(foreach component,$(filter-out dep-% internal-% test-%,$(COMPONENTS)),$(component).wasm $(component).debug.wasm) + +.PHONY: publish ## Publish each component in the target/components directory, `make publish-` publishes one, e.g. publish-gate.wasm +publish: $(addprefix publish-,$(PUBLISH_FILES)) + +.PHONY: $(addprefix publish-,$(PUBLISH_FILES)) +$(addprefix publish-,$(PUBLISH_FILES)): publish-%: | $(call tool,wkg) + @VERSION="$(VERSION)" REPOSITORY="$(REPOSITORY)" COMPONENTS_DIR="$(COMPONENTS_DIR)" SIGN="$(SIGN)" PUBLISH_LOG="$(PUBLISH_LOG)" DRY_RUN="$(DRY_RUN)" \ + build/scripts/publish.sh $* diff --git a/scripts/bump-version.sh b/build/scripts/bump-version.sh similarity index 50% rename from scripts/bump-version.sh rename to build/scripts/bump-version.sh index 4d66d84..6fdf0b6 100755 --- a/scripts/bump-version.sh +++ b/build/scripts/bump-version.sh @@ -1,22 +1,30 @@ #!/usr/bin/env bash -# Bump the version of the wit interface package. +# Bump the version of the wit interface package, and of the crates. # -# scripts/bump-version.sh +# build/scripts/bump-version.sh # # Updates the package declaration and every reference to the package in tracked files, then -# refreshes the generated wit dependencies. Items whose `@since` names an unreleased (prerelease) +# refreshes the generated wit dependencies. The crates share the interface's version: the +# workspace version the crates inherit, and the workspace's requirements on its own crates that name +# a version, e.g. `componentized-constants = { path = "./crates/componentized-constants", version = +# "=0.2.1-dev" }`, move to the new version too. A repository without a workspace version only bumps the +# interface. The default version of the bump-version workflow moves to the new version, the next bump +# is offered from it. Items whose `@since` names an unreleased (prerelease) # version move to the new version, since they were never published under the old one. Items # released under the old version keep their `@since`. # # 0.1.0-dev -> 0.1.0 releases 0.1.0, `@since(version = 0.1.0-dev)` becomes 0.1.0 # 0.1.0 -> 0.2.0-dev starts 0.2.0, `@since(version = 0.1.0)` is unchanged +# +# PACKAGE the interface package, the first package declared in wit/worlds.wit by default, e.g. +# componentized:valkey set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -PACKAGE="${PACKAGE:-componentized:$(basename $(git rev-parse --show-toplevel))}" +PACKAGE="${PACKAGE:-$(sed -n 's/^package \([^@]*\)@.*;$/\1/p' wit/worlds.wit | head -n 1)}" SEMVER='^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$' new="${1:-}" @@ -39,6 +47,7 @@ version_lt() { [[ "$2" == *-* ]] && b_pre="${2#*-}" if [[ "$a_core" != "$b_core" ]]; then local IFS=. + # shellcheck disable=SC2206 # split on the dots local -a a=($a_core) b=($b_core) for i in 0 1 2; do (( a[i] < b[i] )) && return 0 @@ -55,22 +64,49 @@ if ! version_lt "$old" "$new"; then exit 1 fi +old_re="${old//./\\.}" + # the bump-version workflow offers the current version as the default for the next bump, checked # before changing anything workflow=.github/workflows/bump-version.yaml -workflow_default="default: \"${old}\" # the current version, kept current by scripts/bump-version.sh" +workflow_default="default: \"${old}\" # the current version, kept current by build/scripts/bump-version.sh" if ! grep -qF "$workflow_default" "$workflow"; then echo "unable to find the current version as the default in ${workflow}, expected: ${workflow_default}" >&2 exit 1 fi -old_re="${old//./\\.}" +# the version in the [workspace.package] section, inherited by the crates +workspace_version=$(sed -n '/^\[workspace\.package\]/,/^\[/s/^version = "\(.*\)"$/\1/p' Cargo.toml 2> /dev/null || true) +# the workspace's requirements on its own crates that name a version, e.g. a cli requiring the +# library at exactly the current version, publishing fails without a version requirement +crate_re='^[a-z0-9_-]+ = \{.*\bpath = "\./crates/[^"]*".*\bversion = "' +if [[ -n "$workspace_version" ]]; then + if [[ "$workspace_version" != "$old" ]]; then + echo "the workspace version ${workspace_version} in Cargo.toml does not match the interface version ${old}" >&2 + exit 1 + fi + # checked before changing anything + requirements=$(grep -E "$crate_re" Cargo.toml | grep -vE "\bversion = \"=?${old_re}\"" || true) + if [[ -n "$requirements" ]]; then + echo "unable to bump requirements in Cargo.toml that are not on the current version ${old}:" >&2 + echo "$requirements" >&2 + exit 1 + fi +fi + # references to the package or one of its interfaces, an interface named for a keyword is escaped -# with `%`, e.g. `componentized:cli/imports@0.2.1-dev` +# with `%`, e.g. `componentized:constants/%u8@0.2.1-dev` ref_re="${PACKAGE}(/%?[a-z0-9-]+)?" # the fetched wit dependencies and the wkg.lock files are left to `make wit`, wkg replaces the -# dependencies and updates the locks for the new version -files=$(git grep --untracked -l -E "${ref_re}@${old_re}" -- ':(exclude,glob)**/wit/deps/**' ':(exclude,glob)**/wkg.lock' || true) +# dependencies and updates the locks for the new version. The files synced from componentized/build +# are left alone, an example in them may name the package. +synced=() +if [[ -f build/MANIFEST ]]; then + while read -r _ path; do + synced+=(":(exclude,literal)${path}") + done < build/MANIFEST +fi +files=$(git grep --untracked -l -E "${ref_re}@${old_re}" -- ':(exclude,glob)**/wit/deps/**' ':(exclude,glob)**/wkg.lock' ${synced[@]+"${synced[@]}"} || true) for file in $files; do sed -i.bak -E "s#(${ref_re})@${old_re}#\1@${new}#g" "$file" rm "$file.bak" @@ -86,10 +122,19 @@ if [[ "$old" == *-* ]]; then done fi +if [[ -n "$workspace_version" ]]; then + perl -pi -e 'if (/^\[workspace\.package\]/ .. /^\[(?!workspace\.package\])/) { s/^version = "[^"]*"/version = "'"${new}"'"/ }' Cargo.toml + echo "updated the workspace version in Cargo.toml" + for crate in $(grep -E "$crate_re" Cargo.toml | sed 's/ .*//'); do + perl -pi -e 's/^(\Q'"${crate}"'\E = \{.*\bversion = "=?)[^"]*(")/${1}'"${new}"'${2}/' Cargo.toml + echo "updated the ${crate} requirement in Cargo.toml" + done +fi + perl -pi -e 's{^(\s+)\Q'"${workflow_default}"'\E$}{${1}'"${workflow_default/\"${old}\"/\"${new}\"}"'}' "$workflow" echo "updated the default version in ${workflow}" # regenerate the wit dependencies for the new version -make wit components test +make generate components test echo "bumped ${PACKAGE} from ${old} to ${new}" diff --git a/build/scripts/check-build.sh b/build/scripts/check-build.sh new file mode 100755 index 0000000..def1824 --- /dev/null +++ b/build/scripts/check-build.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Check the files synced from componentized/build are unchanged since build/scripts/update-build.sh +# synced them, the version in build/VERSION. +# +# build/scripts/check-build.sh +# +# Fails when a synced file was changed or removed, or a file was added to build/. A change to the +# build belongs in componentized/build, released and synced to each repository, rather than here. +# The current version the repository keeps in a synced file, the default of the bump-version +# workflow, is its own, and may change. + +set -euo pipefail + +cd "$(dirname "$0")/../.." + +# the line of a synced file the repository keeps current, the default version of the bump-version +# workflow, e.g. `default: "0.1.0-dev" # the current version, kept current by +# build/scripts/bump-version.sh`. Its value is the repository's own, left out of the checksum. +CURRENT_VERSION_RE='"[^"]*"( # the current version, kept current by build/scripts/bump-version\.sh)$' +# the checksum of a synced file, without the value the repository keeps current +checksum() { + sed -E "s|${CURRENT_VERSION_RE}|\"\"\1|" "$1" | shasum -a 256 | cut -d ' ' -f 1 +} + +# componentized/build itself, the source of the files rather than a copy of them +if [[ -f SYNC ]]; then + echo "the build repository, nothing synced to check" + exit 0 +fi + +if [[ ! -f build/MANIFEST || ! -f build/VERSION ]]; then + echo "unable to find build/MANIFEST and build/VERSION, run \`make update-build\` to sync the build" >&2 + exit 1 +fi +version=$(cat build/VERSION) + +status=0 +while read -r sum path; do + if [[ ! -f "$path" ]]; then + echo "${path}: removed" + status=1 + elif [[ "$(checksum "$path")" != "$sum" ]]; then + echo "${path}: changed" + status=1 + fi +done < build/MANIFEST + +# build/ only holds synced files +while read -r path; do + case "$path" in + build/VERSION | build/MANIFEST) ;; + *) + if ! grep -qE "^[0-9a-f]{64} ${path//./\\.}$" build/MANIFEST; then + echo "${path}: not synced from componentized/build ${version}" + status=1 + fi + ;; + esac +done < <(find build -type f | sort) + +if [[ "$status" != 0 ]]; then + echo "the build differs from componentized/build ${version}. Make the change in componentized/build and update to its release, or run \`make update-build BUILD_VERSION=${version}\` to undo it." >&2 +fi +exit "$status" diff --git a/build/scripts/create-commit.sh b/build/scripts/create-commit.sh new file mode 100755 index 0000000..b0c0e1e --- /dev/null +++ b/build/scripts/create-commit.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Create a commit of the staged changes with the GitHub REST API, signed by GitHub, for a GitHub App +# token that can't push. +# +# GH_TOKEN= APP_SLUG= build/scripts/create-commit.sh +# +# Prints the commit, a child of , the commit the changes are staged against. No branch is moved +# to it. The changes are only staged locally to find the changed files and their modes. +# +# The commit is authored, committed and signed off (DCO) by the app's bot. GitHub only signs commits +# it attributes entirely to the app, so the author and committer are left for GitHub to fill in. The +# user who triggered the workflow is credited as a co-author, with their GitHub noreply email so their +# email address isn't exposed. A run started by the app has no one else to credit. +# +# GH_TOKEN a token for the app, with contents write permission, and workflows write +# permission to change workflow files +# APP_SLUG the slug of the app, e.g. custodian +# GITHUB_REPOSITORY the repository to create the commit in, e.g. componentized/http +# GITHUB_ACTOR the user who triggered the workflow, credited as a co-author +# GITHUB_ACTOR_ID the id of the user +# RUNNER_TEMP a directory for temporary files + +set -euo pipefail + +base="${1:?usage: $0 }" +subject="${2:?usage: $0 }" +: "${GH_TOKEN:?GH_TOKEN is undefined}" +: "${APP_SLUG:?APP_SLUG is undefined}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is undefined}" + +api="repos/${GITHUB_REPOSITORY}" +temp="${RUNNER_TEMP:-$(mktemp -d)}" + +# a blob for each changed file, or a null sha for a deleted file +entries="${temp}/tree-entries.json" +echo '[]' > "${entries}" +git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do + if [ "${status}" = "D" ] ; then + entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' ) + else + mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 ) + sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha ) + entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' ) + fi + jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}" + echo "${status} ${path}" >&2 +done +tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha ) + +bot="${APP_SLUG}[bot]" +bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com" +message=$( printf '%s\n\nSigned-off-by: %s <%s>' "${subject}" "${bot}" "${bot_email}" ) +if [ -n "${GITHUB_ACTOR:-}" ] && [ "${GITHUB_ACTOR}" != "${bot}" ] ; then + name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' ) + name="${name:-${GITHUB_ACTOR}}" + email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + message=$( printf '%s\nCo-authored-by: %s <%s>' "${message}" "${name}" "${email}" ) +fi +read -r commit verified < <( jq -n \ + --arg message "${message}" \ + --arg tree "${tree}" --arg parent "${base}" \ + '{message: $message, tree: $tree, parents: [$parent]}' \ + | gh api --method POST "${api}/git/commits" --input - --jq '"\(.sha) \(.verification.verified)"' ) +echo "created commit ${commit}" >&2 +# the branch requires signed commits, fail before pushing rather than after +if [ "${verified}" != "true" ] ; then + echo "::error::the commit '${commit}' was not created, or was not signed by GitHub" >&2 + exit 1 +fi + +echo "${commit}" diff --git a/scripts/publish.sh b/build/scripts/publish.sh similarity index 59% rename from scripts/publish.sh rename to build/scripts/publish.sh index 669c112..5771999 100755 --- a/scripts/publish.sh +++ b/build/scripts/publish.sh @@ -2,13 +2,14 @@ # Publish a built component to an OCI registry, and sign it with cosign. # -# VERSION= REPOSITORY= scripts/publish.sh +# VERSION= REPOSITORY= build/scripts/publish.sh # # The file is one of the files built into the components directory, components are in a directory # of their own, the interface is not, e.g. gate.wasm from target/components/gate/gate.wasm and # interface.wasm from target/components/interface.wasm. The interface is published to the # repository, each component to a repository of its own under it, a debug build is tagged with a -# `_debug` suffix. +# `_debug` suffix. A debug build identical to the release build adds nothing, e.g. components +# without debug info, and is not published. # # interface.wasm -> ${REPOSITORY}:0.1.0 # gate.wasm -> ${REPOSITORY}/gate:0.1.0 @@ -20,12 +21,14 @@ # COMPONENTS_DIR the directory the components are built into, defaults to target/components # SIGN sign the published components with cosign, `false` to push without signing, # e.g. to a local registry -# PUBLISH_LOG append the published file and its image to this file, e.g. -# `gate.wasm ghcr.io/componentized/http/gate:0.1.0@sha256:...` +# PUBLISH_LOG append the published file, its size in bytes and its image to this file, e.g. +# `gate.wasm 1234567 ghcr.io/componentized/http/gate:0.1.0@sha256:...` +# DRY_RUN `true` to show what would be published without pushing or signing anything, +# the image is logged with a `dry-run` digest set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." file="${1:-}" if [[ -z "$file" ]]; then @@ -38,6 +41,7 @@ GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-componentized/$(basename "$(git rev-pars COMPONENTS_DIR="${COMPONENTS_DIR:-target/components}" SIGN="${SIGN:-true}" PUBLISH_LOG="${PUBLISH_LOG:-}" +DRY_RUN="${DRY_RUN:-false}" component="${file%.wasm}" component="${component%.debug}" @@ -53,6 +57,11 @@ else fi [[ -n "$debug" ]] && title="${title} (debug)" +if [[ -n "$debug" ]] && cmp -s "${COMPONENTS_DIR}/${component_file}" "${COMPONENTS_DIR}/${component}/${component}.wasm"; then + echo "Not publishing ${file} as it is identical to ${component}.wasm" + exit 0 +fi + # the description is the line following the title in the readme readme="${COMPONENTS_DIR}/$(dirname "$component_file")/README.md" description=$(sed -n 3p "$readme") @@ -75,32 +84,44 @@ else fi echo "::group::${file} -> ${image}" +# close the group when the push or signing fails, so the error isn't hidden in it +trap 'echo "::endgroup::"' EXIT -digest=$( - wkg oci push \ - --annotation "org.opencontainers.image.title=${title}" \ - --annotation "org.opencontainers.image.description=${description}" \ - --annotation "org.opencontainers.image.version=${component_version}" \ - --annotation "org.opencontainers.image.url=${url}" \ - --annotation "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git" \ - --annotation "org.opencontainers.image.revision=${revision}" \ - --annotation "org.opencontainers.image.licenses=Apache-2.0" \ - "$image" \ - "${COMPONENTS_DIR}/${component_file}" \ - 2>&1 \ - | tee /dev/stderr \ - | grep -o 'sha256:[a-f0-9]\{64\}' +annotations=( + --annotation="org.opencontainers.image.title=${title}" + --annotation="org.opencontainers.image.description=${description}" + --annotation="org.opencontainers.image.version=${component_version}" + --annotation="org.opencontainers.image.url=${url}" + --annotation="org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git" + --annotation="org.opencontainers.image.revision=${revision}" + --annotation="org.opencontainers.image.licenses=Apache-2.0" ) +if [[ "$DRY_RUN" == true ]]; then + echo "Not pushing ${COMPONENTS_DIR}/${component_file} to ${image}, DRY_RUN=${DRY_RUN}" + printf ' %s\n' "${annotations[@]}" + digest=dry-run +else + digest=$( + wkg oci push \ + "${annotations[@]}" \ + "$image" \ + "${COMPONENTS_DIR}/${component_file}" \ + 2>&1 \ + | tee /dev/stderr \ + | grep -o 'sha256:[a-f0-9]\{64\}' + ) +fi + if [[ -n "$PUBLISH_LOG" ]]; then size=$(wc -c < "${COMPONENTS_DIR}/${component_file}" | tr -d ' ') echo "${file} ${size} ${image}@${digest}" >> "$PUBLISH_LOG" fi -if [[ "$SIGN" == true ]]; then +if [[ "$DRY_RUN" == true ]]; then + echo "Not signing ${image}@${digest}, DRY_RUN=${DRY_RUN}" +elif [[ "$SIGN" == true ]]; then cosign sign --yes "${image}@${digest}" else echo "Not signing ${image}@${digest}, SIGN=${SIGN}" fi - -echo "::endgroup::" diff --git a/build/scripts/update-build.sh b/build/scripts/update-build.sh new file mode 100755 index 0000000..2d4c258 --- /dev/null +++ b/build/scripts/update-build.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +# Update the build synced from componentized/build to one of its releases. +# +# build/scripts/update-build.sh [] +# +# Downloads the release, removes the files synced from the previous version, listed in +# build/MANIFEST, copies in the files of the release, listed in its SYNC file, and records the +# version in build/VERSION and a checksum of each file in build/MANIFEST, checked by +# build/scripts/check-build.sh. The current version the repository keeps in a synced file, the +# default of the bump-version workflow, is carried forward. A tool pinned in tools/Cargo.toml to a version older than the +# release's build/tool-versions requires is raised to that version. +# +# the release to update to, e.g. v0.2.0, the latest release by default +# BUILD_REPOSITORY the GitHub repository of the build, defaults to componentized/build +# BUILD_SOURCE a checkout of the build repository to update from in place of a release, e.g. to +# try a change before releasing it, recorded as its commit in build/VERSION +# GH_TOKEN a GitHub token for the API, optional, raises its rate limit + +set -euo pipefail + +# the update replaces this script, bash reads a script while it runs it, so the whole script is in a +# block bash reads before running any of it +{ + +cd "$(dirname "$0")/../.." + +BUILD_REPOSITORY="${BUILD_REPOSITORY:-componentized/build}" +BUILD_SOURCE="${BUILD_SOURCE:-}" +version="${1:-}" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +if [[ -n "$BUILD_SOURCE" ]]; then + source=$(cd "$BUILD_SOURCE" && pwd) + if [[ -z "$version" ]]; then + version="$(git -C "$source" rev-parse --verify --quiet HEAD || echo local)" + [[ -n "$(git -C "$source" status --porcelain 2> /dev/null)" ]] && version="${version}+dirty" + fi +else + auth=() + [[ -n "${GH_TOKEN:-}" ]] && auth=(--header "Authorization: Bearer ${GH_TOKEN}") + if [[ -z "$version" ]]; then + version=$(curl -fsSL ${auth[@]+"${auth[@]}"} "https://api.github.com/repos/${BUILD_REPOSITORY}/releases/latest" | jq -r .tag_name) + fi + if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "the version '${version}' is not a release of ${BUILD_REPOSITORY}, e.g. v0.2.0" >&2 + exit 1 + fi + source="$work/source" + mkdir -p "$source" + echo "downloading ${BUILD_REPOSITORY} ${version}" + curl -fsSL "https://github.com/${BUILD_REPOSITORY}/archive/refs/tags/${version}.tar.gz" | tar -xz --strip-components 1 -C "$source" +fi + +if [[ ! -f "$source/SYNC" ]]; then + echo "unable to find the SYNC file of ${BUILD_REPOSITORY} ${version}" >&2 + exit 1 +fi + +# the line of a synced file the repository keeps current, the default version of the bump-version +# workflow, e.g. `default: "0.1.0-dev" # the current version, kept current by +# build/scripts/bump-version.sh`. Its value is the repository's own, left out of the checksum. +CURRENT_VERSION_RE='"[^"]*"( # the current version, kept current by build/scripts/bump-version\.sh)$' +# the checksum of a synced file, without the value the repository keeps current +checksum() { + sed -E "s|${CURRENT_VERSION_RE}|\"\"\1|" "$1" | shasum -a 256 | cut -d ' ' -f 1 +} + +# the current version kept in a file, e.g. `0.1.0-dev`, also from a file of the build before it moved +# into build/, kept current by scripts/bump-version.sh +current_version() { + [[ -f "$1" ]] && sed -nE 's|.*"([^"]*)" # the current version, kept current by (build/)?scripts/bump-version\.sh$|\1|p' "$1" | head -n 1 +} + +# the repository's version, the version of its interface package, e.g. `package componentized:http@0.1.0-dev;` +interface_version=$(sed -n 's/^package [^@]*@\(.*\);$/\1/p' wit/worlds.wit 2> /dev/null | head -n 1 || true) + +# a path in the repository, relative and inside it, e.g. build/build.mk +check_path() { + if [[ "$1" == /* || "/$1/" == */../* ]]; then + echo "refusing to sync '$1', outside the repository" >&2 + exit 1 + fi +} + +# the files of the release, each path in SYNC is a file or a directory of files +synced="$work/synced" +: > "$synced" +while read -r path; do + [[ -z "$path" || "$path" == \#* ]] && continue + check_path "$path" + if [[ ! -e "$source/$path" ]]; then + echo "unable to find ${path}, listed in the SYNC file of ${BUILD_REPOSITORY} ${version}" >&2 + exit 1 + fi + # a checkout syncs only its tracked and unignored files + if git -C "$source" rev-parse --is-inside-work-tree > /dev/null 2>&1; then + git -C "$source" ls-files --cached --others --exclude-standard -- "$path" >> "$synced" + else + (cd "$source" && find "$path" -type f) >> "$synced" + fi +done < "$source/SYNC" +sort -u -o "$synced" "$synced" + +# the files synced from the previous version, a file the release no longer has is removed +if [[ -f build/MANIFEST ]]; then + while read -r _ path; do + check_path "$path" + if ! grep -qxF "$path" "$synced"; then + rm -f "$path" + echo "removed ${path}" + fi + done < build/MANIFEST +fi + +while read -r path; do + mkdir -p "$(dirname "$path")" + cp "$source/$path" "$path.tmp" + # the repository's current version rather than the build's, or its interface version for a file + # it didn't have yet + current=$(current_version "$path" || true) + current="${current:-$interface_version}" + if [[ -n "$current" ]]; then + sed -E "s|${CURRENT_VERSION_RE}|\"${current}\"\1|" "$path.tmp" > "$path.tmp.current" + mv "$path.tmp.current" "$path.tmp" + fi + if cmp -s "$path.tmp" "$path"; then + rm "$path.tmp" + else + # moved into place rather than overwritten, e.g. this script while it runs + mv "$path.tmp" "$path" + echo "updated ${path}" + fi + chmod "$( [[ -x "$source/$path" ]] && echo 755 || echo 644 )" "$path" +done < "$synced" + +echo "$version" > build/VERSION +while read -r path; do + echo "$(checksum "$path") ${path}" +done < "$synced" > build/MANIFEST + +# raise the tools pinned to a version older than the build requires, e.g. `wac-cli = "=0.11.0"` +while read -r tool required; do + [[ -z "$tool" || "$tool" == \#* ]] && continue + pinned=$(sed -n "s/^${tool} = \"=\(.*\)\"$/\1/p" tools/Cargo.toml 2> /dev/null || true) + [[ -z "$pinned" ]] && continue + if [[ "$pinned" != "$required" && "$(printf '%s\n%s\n' "$pinned" "$required" | sort -V | head -n 1)" == "$pinned" ]]; then + perl -pi -e 's/^(\Q'"${tool}"'\E = "=)[^"]*(")/${1}'"${required}"'${2}/' tools/Cargo.toml + echo "updated ${tool} from ${pinned} to ${required} in tools/Cargo.toml, required by the build" + fi +done < build/tool-versions + +echo "updated the build to ${BUILD_REPOSITORY} ${version}" +exit +} diff --git a/build/scripts/wac-deps.sh b/build/scripts/wac-deps.sh new file mode 100755 index 0000000..7e10af8 --- /dev/null +++ b/build/scripts/wac-deps.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Encode each WIT package of a wit/deps directory where `wac compose --deps-dir` looks for it, so a +# composition can name the interfaces of a package it does not instantiate, e.g. +# `import stderr: wasi:cli/stderr@0.3.0;`. +# +# WASM_TOOLS= scripts/wac-deps.sh +# +# wac looks for a versioned package at `//.wasm`, e.g. +# +# wit/deps/wasi-cli-0.3.0 -> /wasi/cli/0.3.0.wasm +# +# Each package is encoded with the other packages as its dependencies. + +set -euo pipefail + +deps=$1 +out=$2 +wasm_tools=${WASM_TOOLS:-wasm-tools} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +rm -rf "$out" +mkdir -p "$out" + +for package in "$deps"/*/; do + package=${package%/} + rm -rf "$work/package" + mkdir -p "$work/package/deps" + cp "$package"/*.wit "$work/package/" + for dep in "$deps"/*/; do + dep=${dep%/} + [[ "$dep" == "$package" ]] || cp -R "$dep" "$work/package/deps/" + done + "$wasm_tools" component wit "$work/package" --wasm -o "$work/package.wasm" + + # e.g. `package wasi:cli@0.3.0;` + id=$("$wasm_tools" component wit "$work/package.wasm" | sed -n 's/^package \(.*\);$/\1/p' | head -n 1) + namespace=${id%%:*} + name=${id#*:} + name=${name%%@*} + version=${id#*@} + mkdir -p "$out/$namespace/$name" + cp "$work/package.wasm" "$out/$namespace/$name/$version.wasm" +done diff --git a/build/sync.mk b/build/sync.mk new file mode 100644 index 0000000..10a8c79 --- /dev/null +++ b/build/sync.mk @@ -0,0 +1,10 @@ +# the release of componentized/build to update to, e.g. v0.2.0, the latest release by default +BUILD_VERSION ?= + +.PHONY: update-build ## Update the build to a release of componentized/build, e.g. `make update-build BUILD_VERSION=v0.2.0` +update-build: + build/scripts/update-build.sh $(BUILD_VERSION) + +.PHONY: check-build ## Check the files synced from componentized/build are unchanged since `make update-build` +check-build: + build/scripts/check-build.sh diff --git a/build/tool-versions b/build/tool-versions new file mode 100644 index 0000000..08758d9 --- /dev/null +++ b/build/tool-versions @@ -0,0 +1,6 @@ +# the lowest version of each tool the build works with, `make update-build` raises a tool pinned in +# tools/Cargo.toml to this version when it is pinned to an older one. A tool missing from +# tools/Cargo.toml is left out, the build only needs it for the components that use it. +# +# +wac-cli 0.12.0 diff --git a/build/tools.mk b/build/tools.mk new file mode 100644 index 0000000..7129904 --- /dev/null +++ b/build/tools.mk @@ -0,0 +1,117 @@ +TOOLS_DIR := target/tools/$(shell rustc --print host-tuple) +# absolute, tools also run from other directories, e.g. `cd components && wkg fetch` +TOOLS_BIN := $(abspath $(TOOLS_DIR))/bin +# for the scripts and tests the recipes run, they look up the tools on the PATH +export PATH := $(TOOLS_BIN):$(PATH) + +# the tools pinned in tools/Cargo.toml, generated when it changes, make restarts after generating it. +# Defines TOOLS, TOOLS_PINNED with the version of each tool, e.g. `wkg@0.16.1`, and a variable for +# each binary a tool installs, e.g. `WKG := $(TOOLS_BIN)/wkg`. Recipes run the pinned tools by path, +# make 3.81 runs a simple command without a shell and looks it up on its own PATH, before the export +# above, which would find a tool already on the system. +TOOLS_MK := target/tools/pinned.mk +include $(TOOLS_MK) + +# the tools built from this repository rather than pinned in tools/Cargo.toml, e.g. a cli the +# components are built with, `LOCAL_TOOLS := componentized-constants-cli`. Each is a package of the +# workspace, installed with the pinned tools, and installed again when its sources change, its own or +# those of the workspace packages it depends on. A local tool replaces a pinned tool of the same name. +# Generated when the workspace changes, defines LOCAL_TOOL_PATH_, LOCAL_TOOL_SOURCES_ +# and a variable for each binary the package installs, like the pinned tools, e.g. +# `CONSTANTS := $(TOOLS_BIN)/constants`. +LOCAL_TOOLS ?= +LOCAL_TOOLS_MK := target/tools/local.mk +ifneq ($(strip $(LOCAL_TOOLS)),) +# quietly, the first time make generates it before reading it +-include $(LOCAL_TOOLS_MK) +endif + +# the toolchain pinned in rust-toolchain.toml, named for `cargo install`, otherwise rustup warns that +# the toolchain file overrides the default +RUST_TOOLCHAIN := $(shell sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2> /dev/null) + +# cargo binstall downloads prebuilt binaries, without it the tools are built with cargo install +CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install) + +.PHONY: clean-tools ## Remove the installed tools, installed again by `make tools` +clean-tools: + rm -rf ${TOOLS_DIR} + +# each tool is a dependency of the tools package, the binaries it installs are listed in +# [package.metadata.bins] +$(TOOLS_MK): tools/Cargo.toml build/pinned.jq + @mkdir -p $(@D) + @set -o pipefail; cargo metadata --manifest-path tools/Cargo.toml --no-deps --format-version 1 | jq -r -f build/pinned.jq > $@.tmp + @mv $@.tmp $@ + +# the packages of the workspace, generated again when LOCAL_TOOLS changes. The workspace's members may +# change without changing Cargo.toml or Cargo.lock, `make clean-tools` generates it again. +$(LOCAL_TOOLS_MK): Cargo.toml Cargo.lock $(filter-out $(TOOLS_MK) $(LOCAL_TOOLS_MK),$(MAKEFILE_LIST)) build/local.jq + @mkdir -p $(@D) + @set -o pipefail; cargo metadata --no-deps --format-version 1 | jq -r --arg tools "$(LOCAL_TOOLS)" -f build/local.jq > $@.tmp + @mv $@.tmp $@ + +tool_version = $(if $(filter $(1),$(LOCAL_TOOLS)),local,$(patsubst $(1)@%,%,$(filter $(1)@%,$(TOOLS_PINNED)))) +# a stamp naming the version of a tool installed in $(TOOLS_DIR)/bin, e.g. `wkg@0.16.1`, the binary +# does not say which version it is. Bumping the pinned version names a stamp that does not exist yet, +# so the tool is installed again. A local tool's stamp is `@local`. +tool = $(TOOLS_DIR)/.installed/$(1)@$(call tool_version,$(1)) + +# a tool a recipe runs: a local tool is a prerequisite, what's made with it is made again when it +# changes, a pinned tool only has to be installed first, e.g. `$(call tool_prerequisite,wkg)`. Last +# among the prerequisites, it may be order-only. +tool_prerequisite = $(if $(filter $(1),$(LOCAL_TOOLS)),$(call tool,$(1)),| $(call tool,$(1))) + +.PHONY: tools ## Install the cli tools pinned in tools/Cargo.toml, and those built from the workspace +tools: $(foreach name,$(sort $(TOOLS) $(LOCAL_TOOLS)),$(call tool,$(name))) + +.PHONY: tools-path ## Print the directory of the installed tools for this platform, to add to the PATH +tools-path: + @echo $(TOOLS_BIN) + +define INSTALL_TOOL + +$(call tool,$1): + $(CARGO_INSTALL) --locked --root $(TOOLS_DIR) --version $(call tool_version,$1) $1 + @mkdir -p $$(@D) + @# only the installed version has a stamp, so going back to a previous version installs it again + @rm -f $$(@D)/$1@* + @touch $$@ + +endef + +$(foreach name,$(filter-out $(LOCAL_TOOLS),$(TOOLS)),$(eval $(call INSTALL_TOOL,$(name)))) + +define INSTALL_LOCAL_TOOL + +$(call tool,$1): Cargo.toml Cargo.lock $(wildcard rust-toolchain.toml) $(if $(LOCAL_TOOL_SOURCES_$1),$(shell find $(LOCAL_TOOL_SOURCES_$1) -type f)) + @# forced, make only installs when the sources change, and the binary may belong to another package + cargo $(if $(RUST_TOOLCHAIN),+$(RUST_TOOLCHAIN)) install --force --locked --root $(TOOLS_DIR) --path $(LOCAL_TOOL_PATH_$1) + @mkdir -p $$(@D) + @# going back to the pinned version installs it again + @rm -f $$(@D)/$1@* + @touch $$@ + +endef + +$(foreach name,$(LOCAL_TOOLS),$(eval $(call INSTALL_LOCAL_TOOL,$(name)))) + +ifeq ($(COMPONENT_TARGET),wasm32-wasip3) + +# wasi-sdk, for its experimental sysroot with a wasi-libc built for component model cooperative +# threads, see build/components.mk. Installed with the tools, set WASI_SDK_PATH to use an existing install. +WASI_SDK_VERSION := 34.0 +WASI_SDK_HOST := $(or $(word 2,$(subst :, ,$(filter $(shell rustc --print host-tuple):%,aarch64-apple-darwin:arm64-macos x86_64-apple-darwin:x86_64-macos aarch64-unknown-linux-gnu:arm64-linux x86_64-unknown-linux-gnu:x86_64-linux))),unsupported-host) +WASI_SDK_DIR := $(abspath $(TOOLS_DIR))/wasi-sdk-$(WASI_SDK_VERSION) +WASI_SDK_PATH ?= $(WASI_SDK_DIR) +WASI_SDK := $(TOOLS_DIR)/.installed/wasi-sdk@$(WASI_SDK_VERSION) + +tools: $(WASI_SDK) + +$(WASI_SDK): + @mkdir -p $(WASI_SDK_DIR) $(@D) + @set -o pipefail; curl -fsSL https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(basename $(WASI_SDK_VERSION))/wasi-sdk-$(WASI_SDK_VERSION)-$(WASI_SDK_HOST).tar.gz | tar xz --strip-components 1 -C $(WASI_SDK_DIR) + @rm -f $(@D)/wasi-sdk@* + @touch $@ + +endif diff --git a/build/wit.mk b/build/wit.mk new file mode 100644 index 0000000..756adc9 --- /dev/null +++ b/build/wit.mk @@ -0,0 +1,41 @@ +# a path relative to the root of the repository, e.g. `wit` for `components/../wit` +relpath = $(if $(filter $(CURDIR),$(abspath $(1))),.,$(patsubst $(CURDIR)/%,%,$(abspath $(1)))) + +# directories with a wkg.toml, each fetches the dependencies of its wit directory into wit/deps, +# e.g. `.` and `components` +WKG_DIRS := $(sort $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell find . -name wkg.toml -not -path './target/*' -not -path '*/deps/*'))))) + +# the wit/deps directory of a directory with a wkg.toml, e.g. `wit/deps` for `.` +wit_deps = $(patsubst ./%,%,$(1)/wit/deps) + +WIT_DEPS := $(foreach dir,$(WKG_DIRS),$(call wit_deps,$(dir))) + +# runs wkg with the arguments in the directory with a wkg.toml, e.g. `$(call wkg,components,fetch)`. +# wkg rewrites wkg.lock even when nothing changed, which would leave everything made from the lock +# looking out of date, so the lock is put back as it was, with its time, when its content is unchanged +wkg = cp -p $(1)/wkg.lock $(1)/wkg.lock.orig && \ + (cd $(1) && $(WKG) $(2)); status=$$?; \ + if cmp -s $(1)/wkg.lock.orig $(1)/wkg.lock; then mv $(1)/wkg.lock.orig $(1)/wkg.lock; else rm $(1)/wkg.lock.orig; fi; \ + exit $$status + +.PHONY: clean-wit ## Remove the fetched wit dependencies, fetched again by `make wit` +clean-wit: + rm -rf $(WIT_DEPS) + +.PHONY: wit ## Fetch the wit dependencies of each wkg.toml into its wit/deps +wit: $(WIT_DEPS) + +define FETCH_WIT + +# the dependencies are touched after fetching, newer than the lock they were fetched with even when +# the fetch changed nothing in them +# +# a package overridden with a local path, e.g. `{ path = "../wit" }`, has its dependencies fetched first, +# an override without dependencies of its own is skipped, e.g. `{ path = "./wit/overrides/wasi-muppets" }` +$(call wit_deps,$1): $1/wkg.toml $1/wkg.lock $(shell find $1/wit -type f -name "*.wit" -not -path "*/deps/*") $(filter $(WIT_DEPS),$(foreach path,$(shell sed -n 's/.*path *= *"\(.*\)".*/\1/p' $1/wkg.toml),$(call relpath,$1/$(path))/deps)) | $(call tool,wkg) + $$(call wkg,$1,fetch) + @touch $(call wit_deps,$1) + +endef + +$(foreach dir,$(WKG_DIRS),$(eval $(call FETCH_WIT,$(dir)))) diff --git a/make/components.mk b/make/components.mk deleted file mode 100644 index 012e5b8..0000000 --- a/make/components.mk +++ /dev/null @@ -1,76 +0,0 @@ -COMPONENTS_DIR := target/components - -COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) - -.PHONY: clean-components -clean-components: clean-wit - rm -rf ${COMPONENTS_DIR} - -.PHONY: components -components: ${COMPONENTS_DIR}/interface.wasm $(foreach component,$(COMPONENTS),${COMPONENTS_DIR}/$(component)/$(component).wasm ${COMPONENTS_DIR}/$(component)/$(component).debug.wasm) - -define BUILD_COMPONENT - -.PHONY: components/$1 -components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm - -ifneq ($(wildcard components/$1/wit/$1.constants.wit),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) - $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-constants-cli) - $(CONSTANTS) --wit components/$1/wit -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -else ifneq ($(wildcard components/$1/$1.properties),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) - $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.properties ${COMPONENTS_DIR}/$1/README.md | $(call tool,componentized-static-config-cli) - $(STATIC_CONFIG) -f components/$1/$1.properties -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -else ifneq ($(wildcard components/$1/$1.wac),) - -# the local packages the composition instantiates, e.g. `new local:latch-n2 { ... }` -WAC_DEPS_$1 := $$(shell grep -v '^\s*//' components/$1/$1.wac | grep -oE 'local:[a-z0-9-]+' | sed 's/^local://' | sort -u) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) - $(WAC) compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) -o ${COMPONENTS_DIR}/$1/$1.wasm components/$1/$1.wac - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) - $(WAC) compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm components/$1/$1.wac - -else ifneq ($(wildcard components/$1/$1.wkg),) - -${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) - $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) - $(WKG) oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | tail -1 2> /dev/null) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -# cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources -else ifneq ($(wildcard components/$1/Cargo.toml),) - -${COMPONENTS_DIR}/$1/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) - cargo build -p $1 --target wasm32-unknown-unknown --release - $(WASM_TOOLS) component new target/wasm32-unknown-unknown/release/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.wasm - -${COMPONENTS_DIR}/$1/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) - cargo build --target wasm32-unknown-unknown -p $1 - $(WASM_TOOLS) component new target/wasm32-unknown-unknown/debug/$(subst -,_,$1).wasm -o ${COMPONENTS_DIR}/$1/$1.debug.wasm - -endif - -${COMPONENTS_DIR}/$1/README.md: components/$1/README.md - @mkdir -p ${COMPONENTS_DIR}/$1 - @cp components/$1/README.md ${COMPONENTS_DIR}/$1/README.md - -endef - -$(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component)))) - -${COMPONENTS_DIR}/interface.wasm: wit/deps README.md | $(call tool,wkg) - @mkdir -p ${COMPONENTS_DIR} - $(WKG) build -o ${COMPONENTS_DIR}/interface.wasm - @cp README.md ${COMPONENTS_DIR}/README.md diff --git a/make/publish.mk b/make/publish.mk deleted file mode 100644 index 2b3a472..0000000 --- a/make/publish.mk +++ /dev/null @@ -1,15 +0,0 @@ -# sign published components with cosign, `SIGN=false` to push without signing, e.g. to a local registry -SIGN ?= true -# append each published file and its image to this file, e.g. `gate.wasm ghcr.io/componentized/http/gate:0.1.0@sha256:...` -PUBLISH_LOG ?= - -# the files that can be published, e.g. gate.wasm, published from target/components/gate/gate.wasm -PUBLISH_FILES := interface.wasm $(foreach component,$(filter-out dep-% test-%,$(COMPONENTS)),$(component).wasm $(component).debug.wasm) - -.PHONY: publish ## Publish each component in the target/components directory -publish: $(addprefix publish-,$(PUBLISH_FILES)) - -.PHONY: $(addprefix publish-,$(PUBLISH_FILES)) -$(addprefix publish-,$(PUBLISH_FILES)): publish-%: | $(call tool,wkg) - @VERSION="$(VERSION)" REPOSITORY="$(REPOSITORY)" COMPONENTS_DIR="$(COMPONENTS_DIR)" SIGN="$(SIGN)" PUBLISH_LOG="$(PUBLISH_LOG)" \ - scripts/publish.sh $* diff --git a/make/tools.mk b/make/tools.mk deleted file mode 100644 index f408f38..0000000 --- a/make/tools.mk +++ /dev/null @@ -1,53 +0,0 @@ -TOOLS_DIR := target/tools/$(shell rustc --print host-tuple) -# absolute, tools also run from other directories, e.g. `cd components && wkg fetch` -TOOLS_BIN := $(abspath $(TOOLS_DIR))/bin -# for the scripts and tests the recipes run, they look up the tools on the PATH -export PATH := $(TOOLS_BIN):$(PATH) - -# the tools pinned in tools/Cargo.toml, generated when it changes, make restarts after generating it. -# Defines TOOLS, TOOLS_PINNED with the version of each tool, e.g. `wkg@0.16.1`, and a variable for -# each binary a tool installs, e.g. `WKG := $(TOOLS_BIN)/wkg`. Recipes run the pinned tools by path, -# make 3.81 runs a simple command without a shell and looks it up on its own PATH, before the export -# above, which would find a tool already on the system. -TOOLS_MK := target/tools/pinned.mk -include $(TOOLS_MK) - -# cargo binstall downloads prebuilt binaries, without it the tools are built with cargo install -CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install) - -.PHONY: clean-tools -clean-tools: - rm -rf ${TOOLS_DIR} - -# each tool is a dependency of the tools package, the binaries it installs are listed in -# [package.metadata.bins] -$(TOOLS_MK): tools/Cargo.toml make/pinned.jq - @mkdir -p $(@D) - @set -o pipefail; cargo metadata --manifest-path tools/Cargo.toml --no-deps --format-version 1 | jq -r -f make/pinned.jq > $@.tmp - @mv $@.tmp $@ - -tool_version = $(patsubst $(1)@%,%,$(filter $(1)@%,$(TOOLS_PINNED))) -# a stamp naming the version of a tool installed in $(TOOLS_DIR)/bin, e.g. `wkg@0.16.1`, the binary -# does not say which version it is. Bumping the pinned version names a stamp that does not exist yet, -# so the tool is installed again. -tool = $(TOOLS_DIR)/.installed/$(1)@$(call tool_version,$(1)) - -.PHONY: tools ## Install the cli tools pinned in tools/Cargo.toml -tools: $(foreach name,$(TOOLS),$(call tool,$(name))) - -.PHONY: tools-path ## Print the directory of the installed tools for this platform, to add to the PATH -tools-path: - @echo $(TOOLS_BIN) - -define INSTALL_TOOL - -$(call tool,$1): - $(CARGO_INSTALL) --locked --root $(TOOLS_DIR) --version $(call tool_version,$1) $1 - @mkdir -p $$(@D) - @# only the installed version has a stamp, so going back to a previous version installs it again - @rm -f $$(@D)/$1@* - @touch $$@ - -endef - -$(foreach name,$(TOOLS),$(eval $(call INSTALL_TOOL,$(name)))) diff --git a/make/wit.mk b/make/wit.mk deleted file mode 100644 index ac4971c..0000000 --- a/make/wit.mk +++ /dev/null @@ -1,28 +0,0 @@ -# a path relative to the root of the repository, e.g. `wit` for `components/../wit` -relpath = $(if $(filter $(CURDIR),$(abspath $(1))),.,$(patsubst $(CURDIR)/%,%,$(abspath $(1)))) - -# directories with a wkg.toml, each fetches the dependencies of its wit directory into wit/deps, -# e.g. `.` and `components` -WKG_DIRS := $(sort $(patsubst ./%,%,$(patsubst %/,%,$(dir $(shell find . -name wkg.toml -not -path './target/*' -not -path '*/deps/*'))))) - -# the wit/deps directory of a directory with a wkg.toml, e.g. `wit/deps` for `.` -wit_deps = $(patsubst ./%,%,$(1)/wit/deps) - -WIT_DEPS := $(foreach dir,$(WKG_DIRS),$(call wit_deps,$(dir))) - -.PHONY: clean-wit ## Remove the fetched wit dependencies, fetched again by `make wit` -clean-wit: - rm -rf $(WIT_DEPS) - -.PHONY: wit -wit: $(WIT_DEPS) - -define FETCH_WIT - -# a package overridden with a local path, e.g. `{ path = "../wit" }`, has its dependencies fetched first -$(call wit_deps,$1): $1/wkg.toml $1/wkg.lock $(shell find $1/wit -type f -name "*.wit" -not -path "*/deps/*") $(foreach path,$(shell sed -n 's/.*path *= *"\(.*\)".*/\1/p' $1/wkg.toml),$(call relpath,$1/$(path))/deps) | $(call tool,wkg) - $(if $(filter .,$1),,cd $1 && )$(WKG) fetch - -endef - -$(foreach dir,$(WKG_DIRS),$(eval $(call FETCH_WIT,$(dir)))) diff --git a/scripts/init-devcontainer.sh b/scripts/init-devcontainer.sh deleted file mode 100755 index 59f0186..0000000 --- a/scripts/init-devcontainer.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -# Setup script run once for new devcontainers to init the environment. - -set -euo pipefail - -curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash - -cargo check - -echo "export \"PATH=$(make -s tools-path):${PATH}\"" >> ~/.bashrc -make tools