Release hackbot agents and services by version

.github/workflows/hackbot-prod-deploy.yml

@@ -7,19 +7,14 @@
 # environment: the deploy tool maps one to the other, and no list of names is
 # kept here.
 #
-# Every stage deploy is offered automatically. When an agent's or service's
-# Cloud Build trigger (agent-<name> / service-<name>, in
-# moz-fx-hackbot-nonprod) reports success, `plan` compares what stage runs
-# with what prod runs. If they differ, it pins stage's digests and `promote`
-# waits for approval, then puts on exactly those, unless stage has moved on in
-# the meantime (superseded: the newer build's run asks for its own approval).
-# A build that didn't change stage, such as one waiting for infra, asks
-# nothing. Each run first cancels older runs of the same name still waiting
-# for approval, so there is at most one request per agent or service, always
-# for what stage runs now.
-#
 # From the Actions tab (anyone with write access):
-#   promote   the same, for one name, still behind the approval.
+#   promote   `plan` compares what stage runs with what prod runs. If they
+#             differ, it pins stage's digests and `promote` waits for
+#             approval, then puts on exactly those, unless stage has moved on
+#             in the meantime (superseded). Each run first cancels older runs
+#             of the same name still waiting for approval, so there is at most
+#             one request per agent or service. A service's promote also marks
+#             its Sentry release (<service>@<version>) deployed to production.
 #   rollback  step prod back to its previous recorded deployment, in the
 #             `hackbot-prod-rollback` environment.
 #
@@ -38,13 +33,12 @@
 #                limited to the default branch
 #   vars.HACKBOT_MANIFEST_SYNC_SA     the prod state's manifest_sync_service_account_email output
 #   vars.HACKBOT_PROD_DEPLOY_SA       the prod state's prod_deploy_service_account_email output
+#   secrets.SENTRY_AUTH_TOKEN         in `hackbot-prod-services`, to mark Sentry deploys (optional)
 # The workload identity provider is the platform's organization variable,
 # GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER, so it needs nothing here.
 name: Hackbot prod deploy
 run-name: >-
-  ${{ github.event_name == 'check_run'
-      && format('Offer {0} to prod', github.event.check_run.name)
-      || format('{0} {1} in prod{2}', inputs.action, inputs.name, inputs.dry_run && ' (dry run)' || '') }}
+  ${{ format('{0} {1} in prod{2}', inputs.action, inputs.name, inputs.dry_run && ' (dry run)' || '') }}
 
 on:
   workflow_dispatch:
@@ -82,14 +76,8 @@ on:
 jobs:
   plan:
     if: >-
-      (github.event_name == 'check_run' &&
-       github.event.check_run.app.slug == 'google-cloud-build' &&
-       github.event.check_run.conclusion == 'success' &&
-       (startsWith(github.event.check_run.name, 'agent-') || startsWith(github.event.check_run.name, 'service-')) &&
-       endsWith(github.event.check_run.name, ' (moz-fx-hackbot-nonprod)'))
-      ||
-      (github.event_name == 'workflow_dispatch' && inputs.action == 'promote' &&
-       github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
+      inputs.action == 'promote' &&
+      github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
     runs-on: ubuntu-latest
     permissions:
       actions: write # cancel older runs still waiting for approval
@@ -107,19 +95,8 @@ jobs:
       - name: Name
         id: name
         env:
-          CHECK_NAME: ${{ github.event.check_run.name }}
           INPUT_NAME: ${{ inputs.name }}
-        run: |
-          # The trigger is agent-<name> or service-<name> (Cloud Build names
-          # can't hold a slash); the deployable is agents/<name> or
-          # services/<name>.
-          if [ -n "$INPUT_NAME" ]; then
-            name="$INPUT_NAME"
-          else
-            trigger="${CHECK_NAME% (*}"
-            name="${trigger%%-*}s/${trigger#*-}"
-          fi
-          echo "name=$name" >> "$GITHUB_OUTPUT"
+        run: echo "name=$INPUT_NAME" >> "$GITHUB_OUTPUT"
 
       # They haven't started deploying (that only begins after approval), so
       # cancelling can't interrupt one. Matched by the promote job's name.
@@ -237,6 +214,32 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
           exit "$status"
 
+      # The stage build made the release (<service>@<version>, with its
+      # commits); a promote puts that same release in production.
+      - name: Mark the Sentry release deployed to production
+        if: startsWith(needs.plan.outputs.name, 'services/')
+        env:
+          NAME: ${{ needs.plan.outputs.name }}
+          PROMOTION: ${{ needs.plan.outputs.promotion }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: mozilla
+        run: |
+          if ! grep -q ": deployed" "$RUNNER_TEMP/out.txt"; then
+            echo "Nothing was deployed; no Sentry deploy to mark."
+            exit 0
+          fi
+          if [ -z "$SENTRY_AUTH_TOKEN" ]; then
+            echo "::warning::No SENTRY_AUTH_TOKEN secret in hackbot-prod-services; the production deploy isn't marked in Sentry."
+            exit 0
+          fi
+          version=$(jq -r '.version // empty' <<< "$PROMOTION")
+          if [ -z "$version" ]; then
+            echo "::warning::Stage's record has no version (deployed before versions were recorded); the production deploy isn't marked in Sentry."
+            exit 0
+          fi
+          service="${NAME#services/}"
+          SENTRY_PROJECT="$service" npx --yes @sentry/cli@2 deploys new --release "$service@$version" -e production
+
   rollback:
     if: >-
       github.event_name == 'workflow_dispatch' && inputs.action == 'rollback' &&

agents/cloudbuild.yaml

@@ -1,23 +1,49 @@
-# Build one agent's images and ship them to dev, then stage.
+# Release agents: build their images and deploy them to one environment.
+#
+# Two triggers (webservices-infra, hackbot/tf/modules/resources/build.tf) run
+# this, telling it which environment in _ENV:
+#   stage-agents  a push to the default branch touching an agent's VERSION
+#                 file. Releases every agent whose version isn't
+#                 the one stage runs. Run by hand, it releases every agent
+#                 regardless of version, or those in _AGENTS=a,b.
+#   dev-agents    a push to dev/agents/<agent> or dev/agents/<agent>/<anything>.
+#                 Releases that agent to dev, always.
+# Prod is a separate, approved promotion; nothing here reaches it.
+#
+# `release-plan` decides what to release and `release` deploys it, one agent
+# after another: one failing doesn't stop the rest, and the build fails at the
+# end, naming them.
+#
+# Cloud Build reads every `$NAME` as a substitution, and MUST_MATCH fails the
+# build on unknown ones, so shell variables below are written `$$name`.
 steps:
-  # Before anything else: if an older build of this trigger is still going,
-  # stop here rather than deploy over it.
-  - id: One at a time
+  # Builds of one trigger deploy to one environment: take turns. The wait
+  # allows for an older build's own work; the timeout is that plus this one's.
+  - id: Wait
     name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim
     entrypoint: bash
     args:
       - -c
-      - uv run --frozen --package hackbot-deploy python -m hackbot_deploy check-build "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION"
+      - uv run --frozen --package hackbot-deploy python -m hackbot_deploy wait-turn "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" --timeout 7200
 
-  - id: Shape
+  # /workspace/plan.txt: one "name version previous-commit config-hash broker"
+  # line per agent to release.
+  - id: Plan
     name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim
     entrypoint: bash
+    # The tool reads these from the environment: in a command, a run by hand's
+    # unresolved pusher ("$(push.pusher.name)") would run as a command.
+    env:
+      - BRANCH_NAME=$BRANCH_NAME
+      - AGENTS=${_AGENTS}
+      - PUSHER=${_PUSHER}
     args:
       - -c
-      - uv run --frozen --package hackbot-deploy python -m hackbot_deploy info "${_AGENT}" > /workspace/shape.env
+      - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release-plan --env "${_ENV}" --output /workspace/plan.txt
 
-  # Every image carries the shape it was built for (hackbot.config_hash); the
-  # deploy step only puts it on infrastructure with that shape applied.
+  # Every image carries the shape it was built for (hackbot.config_hash) and
+  # its version. An agent whose images don't build gets no digest files, and
+  # `release` reports it.
   - id: Build
     name: gcr.io/cloud-builders/docker
     entrypoint: bash
@@ -26,25 +52,26 @@ steps:
     args:
       - -c
       - |
-        set -euo pipefail
-        source /workspace/shape.env
-        created=$$(date -u +%Y-%m-%dT%H:%M:%SZ)
-        targets=agent
-        if [ "$$HAS_BROKER" = true ]; then targets="agent broker"; fi
-        for target in $$targets; do
-          image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/${_AGENT}-$$target:$COMMIT_SHA"
-          docker build -f "agents/${_AGENT}/Dockerfile" --target "$$target" \
-            --label "hackbot.config_hash=$$CONFIG_HASH" \
-            --label "org.opencontainers.image.revision=$COMMIT_SHA" \
-            --label "org.opencontainers.image.created=$$created" \
-            -t "$$image" .
-          docker push "$$image"
-          docker inspect --format='{{index .RepoDigests 0}}' "$$image" > "/workspace/$$target.digest"
-        done
+        set -uo pipefail
+        while read -r name version _ config_hash broker; do
+          agent="$${name#agents/}"
+          targets=agent
+          if [ "$$broker" = broker ]; then targets="agent broker"; fi
+          for target in $$targets; do
+            image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/$$agent-$$target:$COMMIT_SHA"
+            if ! { docker build -f "agents/$$agent/Dockerfile" --target "$$target" \
+                     --label "hackbot.config_hash=$$config_hash" \
+                     --label "org.opencontainers.image.version=$$version" \
+                     --label "org.opencontainers.image.revision=$COMMIT_SHA" \
+                     --label "org.opencontainers.image.created=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+                     -t "$$image" . && docker push "$$image"; } < /dev/null; then
+              rm -f "/workspace/$$agent-agent.digest"
+              break
+            fi
+            docker inspect --format='{{index .RepoDigests 0}}' "$$image" > "/workspace/$$agent-$$target.digest"
+          done
+        done < /workspace/plan.txt
 
-  # Dev first, then stage with the same digests. If the shape isn't applied
-  # yet (exit 3), stop without failing: the deploy is rerun once the infra PR
-  # for this change is applied.
   - id: Deploy
     name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim
     entrypoint: bash
@@ -55,28 +82,20 @@ steps:
       - BUILD_ID=$BUILD_ID
     args:
       - -c
-      - |
-        set -uo pipefail
-        images=(--agent-image "$$(cat /workspace/agent.digest)")
-        if [ -f /workspace/broker.digest ]; then images+=(--broker-image "$$(cat /workspace/broker.digest)"); fi
-        for env in dev stage; do
-          uv run --frozen --package hackbot-deploy python -m hackbot_deploy deploy "agents/${_AGENT}" --env "$$env" "$${images[@]}"
-          status=$$?
-          if [ "$$status" -eq 3 ]; then
-            echo "Waiting for infra: not deploying past $$env."
-            exit 0
-          fi
-          if [ "$$status" -ne 0 ]; then exit "$$status"; fi
-        done
+      - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release --env "${_ENV}" --plan /workspace/plan.txt --digests /workspace
 
-# The trigger sets it from the push event; empty when run any other way.
+# _ENV is set by each trigger; _AGENTS only when run by hand; _PUSHER from the
+# push event, empty when run any other way.
 substitutions:
+  _ENV: stage
+  _AGENTS: ""
   _PUSHER: ""
 
+timeout: 14400s
+
 options:
   substitutionOption: MUST_MATCH
   logging: CLOUD_LOGGING_ONLY
-  # Agent images can be large; size the builder for them rather than the
-  # default.
+  # Agent images are large, and one build can release several of them.
   machineType: E2_HIGHCPU_8
-  diskSizeGb: 100
+  diskSizeGb: 200