diff --git a/.github/workflows/hackbot-prod-deploy.yml b/.github/workflows/hackbot-prod-deploy.yml index e2779829d20..e808a0d01ab 100644 --- a/.github/workflows/hackbot-prod-deploy.yml +++ b/.github/workflows/hackbot-prod-deploy.yml @@ -7,19 +7,14 @@ # environment: the deploy tool maps one to the other, and no list of names is # kept here. # -# Every stage deploy is offered automatically. When an agent's or service's -# Cloud Build trigger (agent- / service-, in -# moz-fx-hackbot-nonprod) reports success, `plan` compares what stage runs -# with what prod runs. If they differ, it pins stage's digests and `promote` -# waits for approval, then puts on exactly those, unless stage has moved on in -# the meantime (superseded: the newer build's run asks for its own approval). -# A build that didn't change stage, such as one waiting for infra, asks -# nothing. Each run first cancels older runs of the same name still waiting -# for approval, so there is at most one request per agent or service, always -# for what stage runs now. -# # From the Actions tab (anyone with write access): -# promote the same, for one name, still behind the approval. +# promote `plan` compares what stage runs with what prod runs. If they +# differ, it pins stage's digests and `promote` waits for +# approval, then puts on exactly those, unless stage has moved on +# in the meantime (superseded). Each run first cancels older runs +# of the same name still waiting for approval, so there is at most +# one request per agent or service. A service's promote also marks +# its Sentry release (@) deployed to production. # rollback step prod back to its previous recorded deployment, in the # `hackbot-prod-rollback` environment. # @@ -38,13 +33,12 @@ # limited to the default branch # vars.HACKBOT_MANIFEST_SYNC_SA the prod state's manifest_sync_service_account_email output # vars.HACKBOT_PROD_DEPLOY_SA the prod state's prod_deploy_service_account_email output +# secrets.SENTRY_AUTH_TOKEN in `hackbot-prod-services`, to mark Sentry deploys (optional) # The workload identity provider is the platform's organization variable, # GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER, so it needs nothing here. name: Hackbot prod deploy run-name: >- - ${{ github.event_name == 'check_run' - && format('Offer {0} to prod', github.event.check_run.name) - || format('{0} {1} in prod{2}', inputs.action, inputs.name, inputs.dry_run && ' (dry run)' || '') }} + ${{ format('{0} {1} in prod{2}', inputs.action, inputs.name, inputs.dry_run && ' (dry run)' || '') }} on: workflow_dispatch: @@ -82,14 +76,8 @@ on: jobs: plan: if: >- - (github.event_name == 'check_run' && - github.event.check_run.app.slug == 'google-cloud-build' && - github.event.check_run.conclusion == 'success' && - (startsWith(github.event.check_run.name, 'agent-') || startsWith(github.event.check_run.name, 'service-')) && - endsWith(github.event.check_run.name, ' (moz-fx-hackbot-nonprod)')) - || - (github.event_name == 'workflow_dispatch' && inputs.action == 'promote' && - github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + inputs.action == 'promote' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) runs-on: ubuntu-latest permissions: actions: write # cancel older runs still waiting for approval @@ -107,19 +95,8 @@ jobs: - name: Name id: name env: - CHECK_NAME: ${{ github.event.check_run.name }} INPUT_NAME: ${{ inputs.name }} - run: | - # The trigger is agent- or service- (Cloud Build names - # can't hold a slash); the deployable is agents/ or - # services/. - if [ -n "$INPUT_NAME" ]; then - name="$INPUT_NAME" - else - trigger="${CHECK_NAME% (*}" - name="${trigger%%-*}s/${trigger#*-}" - fi - echo "name=$name" >> "$GITHUB_OUTPUT" + run: echo "name=$INPUT_NAME" >> "$GITHUB_OUTPUT" # They haven't started deploying (that only begins after approval), so # cancelling can't interrupt one. Matched by the promote job's name. @@ -237,6 +214,32 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" exit "$status" + # The stage build made the release (@, with its + # commits); a promote puts that same release in production. + - name: Mark the Sentry release deployed to production + if: startsWith(needs.plan.outputs.name, 'services/') + env: + NAME: ${{ needs.plan.outputs.name }} + PROMOTION: ${{ needs.plan.outputs.promotion }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: mozilla + run: | + if ! grep -q ": deployed" "$RUNNER_TEMP/out.txt"; then + echo "Nothing was deployed; no Sentry deploy to mark." + exit 0 + fi + if [ -z "$SENTRY_AUTH_TOKEN" ]; then + echo "::warning::No SENTRY_AUTH_TOKEN secret in hackbot-prod-services; the production deploy isn't marked in Sentry." + exit 0 + fi + version=$(jq -r '.version // empty' <<< "$PROMOTION") + if [ -z "$version" ]; then + echo "::warning::Stage's record has no version (deployed before versions were recorded); the production deploy isn't marked in Sentry." + exit 0 + fi + service="${NAME#services/}" + SENTRY_PROJECT="$service" npx --yes @sentry/cli@2 deploys new --release "$service@$version" -e production + rollback: if: >- github.event_name == 'workflow_dispatch' && inputs.action == 'rollback' && diff --git a/agents/cloudbuild.yaml b/agents/cloudbuild.yaml index d3902489584..bbe29af39e0 100644 --- a/agents/cloudbuild.yaml +++ b/agents/cloudbuild.yaml @@ -1,23 +1,49 @@ -# Build one agent's images and ship them to dev, then stage. +# Release agents: build their images and deploy them to one environment. +# +# Two triggers (webservices-infra, hackbot/tf/modules/resources/build.tf) run +# this, telling it which environment in _ENV: +# stage-agents a push to the default branch touching an agent's VERSION +# file. Releases every agent whose version isn't +# the one stage runs. Run by hand, it releases every agent +# regardless of version, or those in _AGENTS=a,b. +# dev-agents a push to dev/agents/ or dev/agents//. +# Releases that agent to dev, always. +# Prod is a separate, approved promotion; nothing here reaches it. +# +# `release-plan` decides what to release and `release` deploys it, one agent +# after another: one failing doesn't stop the rest, and the build fails at the +# end, naming them. +# +# Cloud Build reads every `$NAME` as a substitution, and MUST_MATCH fails the +# build on unknown ones, so shell variables below are written `$$name`. steps: - # Before anything else: if an older build of this trigger is still going, - # stop here rather than deploy over it. - - id: One at a time + # Builds of one trigger deploy to one environment: take turns. The wait + # allows for an older build's own work; the timeout is that plus this one's. + - id: Wait name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash args: - -c - - uv run --frozen --package hackbot-deploy python -m hackbot_deploy check-build "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy wait-turn "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" --timeout 7200 - - id: Shape + # /workspace/plan.txt: one "name version previous-commit config-hash broker" + # line per agent to release. + - id: Plan name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash + # The tool reads these from the environment: in a command, a run by hand's + # unresolved pusher ("$(push.pusher.name)") would run as a command. + env: + - BRANCH_NAME=$BRANCH_NAME + - AGENTS=${_AGENTS} + - PUSHER=${_PUSHER} args: - -c - - uv run --frozen --package hackbot-deploy python -m hackbot_deploy info "${_AGENT}" > /workspace/shape.env + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release-plan --env "${_ENV}" --output /workspace/plan.txt - # Every image carries the shape it was built for (hackbot.config_hash); the - # deploy step only puts it on infrastructure with that shape applied. + # Every image carries the shape it was built for (hackbot.config_hash) and + # its version. An agent whose images don't build gets no digest files, and + # `release` reports it. - id: Build name: gcr.io/cloud-builders/docker entrypoint: bash @@ -26,25 +52,26 @@ steps: args: - -c - | - set -euo pipefail - source /workspace/shape.env - created=$$(date -u +%Y-%m-%dT%H:%M:%SZ) - targets=agent - if [ "$$HAS_BROKER" = true ]; then targets="agent broker"; fi - for target in $$targets; do - image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/${_AGENT}-$$target:$COMMIT_SHA" - docker build -f "agents/${_AGENT}/Dockerfile" --target "$$target" \ - --label "hackbot.config_hash=$$CONFIG_HASH" \ - --label "org.opencontainers.image.revision=$COMMIT_SHA" \ - --label "org.opencontainers.image.created=$$created" \ - -t "$$image" . - docker push "$$image" - docker inspect --format='{{index .RepoDigests 0}}' "$$image" > "/workspace/$$target.digest" - done + set -uo pipefail + while read -r name version _ config_hash broker; do + agent="$${name#agents/}" + targets=agent + if [ "$$broker" = broker ]; then targets="agent broker"; fi + for target in $$targets; do + image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/$$agent-$$target:$COMMIT_SHA" + if ! { docker build -f "agents/$$agent/Dockerfile" --target "$$target" \ + --label "hackbot.config_hash=$$config_hash" \ + --label "org.opencontainers.image.version=$$version" \ + --label "org.opencontainers.image.revision=$COMMIT_SHA" \ + --label "org.opencontainers.image.created=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -t "$$image" . && docker push "$$image"; } < /dev/null; then + rm -f "/workspace/$$agent-agent.digest" + break + fi + docker inspect --format='{{index .RepoDigests 0}}' "$$image" > "/workspace/$$agent-$$target.digest" + done + done < /workspace/plan.txt - # Dev first, then stage with the same digests. If the shape isn't applied - # yet (exit 3), stop without failing: the deploy is rerun once the infra PR - # for this change is applied. - id: Deploy name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash @@ -55,28 +82,20 @@ steps: - BUILD_ID=$BUILD_ID args: - -c - - | - set -uo pipefail - images=(--agent-image "$$(cat /workspace/agent.digest)") - if [ -f /workspace/broker.digest ]; then images+=(--broker-image "$$(cat /workspace/broker.digest)"); fi - for env in dev stage; do - uv run --frozen --package hackbot-deploy python -m hackbot_deploy deploy "agents/${_AGENT}" --env "$$env" "$${images[@]}" - status=$$? - if [ "$$status" -eq 3 ]; then - echo "Waiting for infra: not deploying past $$env." - exit 0 - fi - if [ "$$status" -ne 0 ]; then exit "$$status"; fi - done + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release --env "${_ENV}" --plan /workspace/plan.txt --digests /workspace -# The trigger sets it from the push event; empty when run any other way. +# _ENV is set by each trigger; _AGENTS only when run by hand; _PUSHER from the +# push event, empty when run any other way. substitutions: + _ENV: stage + _AGENTS: "" _PUSHER: "" +timeout: 14400s + options: substitutionOption: MUST_MATCH logging: CLOUD_LOGGING_ONLY - # Agent images can be large; size the builder for them rather than the - # default. + # Agent images are large, and one build can release several of them. machineType: E2_HIGHCPU_8 - diskSizeGb: 100 + diskSizeGb: 200 diff --git a/services/hackbot-api/cloudbuild.yaml b/services/hackbot-api/cloudbuild.yaml index a285c84f78c..c99a19dc98d 100644 --- a/services/hackbot-api/cloudbuild.yaml +++ b/services/hackbot-api/cloudbuild.yaml @@ -1,23 +1,49 @@ -# Build hackbot-api once and ship it to dev, then stage. +# Release hackbot-api: build it and deploy it to one environment. # -# Its trigger (webservices-infra, hackbot/tf/modules/resources/build.tf) runs -# this on a push to the default branch touching the API or the libraries it's -# built from. Prod is a separate, approved promotion; nothing here reaches it. +# Two triggers (webservices-infra, hackbot/tf/modules/resources/build.tf) run +# this, telling it which environment in _ENV: +# stage-hackbot-api a push to the default branch touching +# services/hackbot-api/VERSION. Deploys only +# if its version isn't the one stage runs; run by hand, it +# deploys regardless. +# dev-hackbot-api a push to dev/services/hackbot-api or +# dev/services/hackbot-api/. Deploys to dev, +# always. +# Prod is a separate, approved promotion; nothing here reaches it. # services/cloudbuild.yaml is hackbot-dev's older config and goes away at # cutover. # +# `release-plan` decides whether to release and `release` deploys it. The +# Sentry release is hackbot-api@ (dev builds add +), with the +# commits since what the environment ran before, and a deploy in its +# environment; prod's deploy is marked by the prod workflow. +# # Cloud Build reads every `$NAME` as a substitution, and MUST_MATCH fails the # build on unknown ones, so shell variables below are written `$$name`. steps: - # Before anything else: if an older build of this trigger is still going, - # stop here rather than deploy over it. - - id: One at a time + # Builds of one trigger deploy to one environment: take turns. The wait + # allows for an older build's own work; the timeout is that plus this one's. + - id: Wait + name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim + entrypoint: bash + args: + - -c + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy wait-turn "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" --timeout 3600 + + # /workspace/plan.txt: "name version previous-commit - -", or empty when + # there is nothing to release (every step below then does nothing). + - id: Plan name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash + # The tool reads these from the environment: in a command, a run by hand's + # unresolved pusher ("$(push.pusher.name)") would run as a command. + env: + - BRANCH_NAME=$BRANCH_NAME + - PUSHER=${_PUSHER} args: - -c - - uv run --frozen --package hackbot-deploy python -m hackbot_deploy check-build "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release-plan services/hackbot-api --env "${_ENV}" --output /workspace/plan.txt - id: Build name: gcr.io/cloud-builders/docker @@ -28,18 +54,22 @@ steps: - -c - | set -euo pipefail + read -r _ version _ < /workspace/plan.txt || exit 0 + release="hackbot-api@$$version" + sha="$COMMIT_SHA" + if [ "${_ENV}" = dev ]; then release="$$release+$${sha:0:12}"; fi image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/hackbot-api:$COMMIT_SHA" docker build --no-cache -f services/hackbot-api/Dockerfile \ - --build-arg "SENTRY_RELEASE=hackbot-api@$COMMIT_SHA" \ + --build-arg "SENTRY_RELEASE=$$release" \ + --label "org.opencontainers.image.version=$$version" \ --label "org.opencontainers.image.revision=$COMMIT_SHA" \ --label "org.opencontainers.image.created=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ -t "$$image" . docker push "$$image" - docker inspect --format='{{index .RepoDigests 0}}' "$$image" > /workspace/image.digest + docker inspect --format='{{index .RepoDigests 0}}' "$$image" > /workspace/hackbot-api.digest - # Same digest to dev, then stage. Services have no shape gate: their - # infrastructure is written by hand in webservices-infra and changes there - # first. + # Services have no shape gate: their infrastructure is written by hand in + # webservices-infra and changes there first. - id: Deploy name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash @@ -50,35 +80,41 @@ steps: - BUILD_ID=$BUILD_ID args: - -c - - | - set -euo pipefail - for env in dev stage; do - uv run --frozen --package hackbot-deploy python -m hackbot_deploy deploy services/hackbot-api --env "$$env" --image "$$(cat /workspace/image.digest)" - done + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release --env "${_ENV}" --plan /workspace/plan.txt --digests /workspace # After the deploy, so a Sentry outage can't hold one up. - - id: Sentry release - name: getsentry/sentry-cli - secretEnv: - - SENTRY_AUTH_TOKEN - args: ["releases", "new", "hackbot-api@$COMMIT_SHA", "--finalize"] - - id: Sentry commits + - id: Sentry name: getsentry/sentry-cli + entrypoint: sh secretEnv: - SENTRY_AUTH_TOKEN args: - [ - "releases", - "set-commits", - "hackbot-api@$COMMIT_SHA", - "--commit", - "mozilla/bugbug@$COMMIT_SHA", - ] + - -c + - | + set -eu + read -r _ version previous _ < /workspace/plan.txt || exit 0 + release="hackbot-api@$$version" + environment=staging + sha="$COMMIT_SHA" + if [ "${_ENV}" = dev ]; then + release="$$release+$$(echo "$$sha" | cut -c1-12)" + environment=development + fi + commits="mozilla/bugbug@$COMMIT_SHA" + if [ "$$previous" != - ]; then commits="mozilla/bugbug@$$previous..$COMMIT_SHA"; fi + sentry-cli releases new "$$release" + sentry-cli releases set-commits "$$release" --commit "$$commits" + sentry-cli releases finalize "$$release" + sentry-cli deploys new --release "$$release" -e "$$environment" -# The trigger sets it from the push event; empty when run any other way. +# _ENV is set by each trigger; _PUSHER from the push event, empty when run +# any other way. substitutions: + _ENV: stage _PUSHER: "" +timeout: 5400s + options: substitutionOption: MUST_MATCH logging: CLOUD_LOGGING_ONLY diff --git a/services/hackbot-ui/cloudbuild.yaml b/services/hackbot-ui/cloudbuild.yaml index 1298208c466..e3dd0d08f56 100644 --- a/services/hackbot-ui/cloudbuild.yaml +++ b/services/hackbot-ui/cloudbuild.yaml @@ -1,27 +1,54 @@ -# Build hackbot-ui once and ship it to dev, then stage. +# Release hackbot-ui: build it and deploy it to one environment. # -# Its trigger (webservices-infra, hackbot/tf/modules/resources/build.tf) runs -# this on a push to the default branch touching the Console. Prod is a -# separate, approved promotion; nothing here reaches it. services/cloudbuild.yaml -# is hackbot-dev's older config and goes away at cutover. +# Two triggers (webservices-infra, hackbot/tf/modules/resources/build.tf) run +# this, telling it which environment in _ENV: +# stage-hackbot-ui a push to the default branch touching +# services/hackbot-ui/VERSION. Deploys only +# if its version isn't the one stage runs; run by hand, it +# deploys regardless. +# dev-hackbot-ui a push to dev/services/hackbot-ui or +# dev/services/hackbot-ui/. Deploys to dev, +# always. +# Prod is a separate, approved promotion; nothing here reaches it. +# services/cloudbuild.yaml is hackbot-dev's older config and goes away at +# cutover. +# +# `release-plan` decides whether to release and `release` deploys it. The +# Sentry release is hackbot-ui@ (dev builds add +), with the +# commits since what the environment ran before, and a deploy in its +# environment; prod's deploy is marked by the prod workflow. +# +# The Console bakes its Sentry DSN and release into the bundle at build time +# (_SENTRY_DSN) and uploads its source maps then, which needs the Sentry token +# as a build secret. It picks its Sentry environment at runtime from its +# hostname, so one image serves every environment. # # Cloud Build reads every `$NAME` as a substitution, and MUST_MATCH fails the # build on unknown ones, so shell variables below are written `$$name`. -# -# The Console bakes its Sentry DSN into the bundle at build time -# (_SENTRY_DSN), and picks its Sentry environment at runtime from its hostname, -# so one image serves every environment. The build also uploads source maps, -# which needs the Sentry token as a build secret. steps: - # Before anything else: if an older build of this trigger is still going, - # stop here rather than deploy over it. - - id: One at a time + # Builds of one trigger deploy to one environment: take turns. The wait + # allows for an older build's own work; the timeout is that plus this one's. + - id: Wait name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash args: - -c - - uv run --frozen --package hackbot-deploy python -m hackbot_deploy check-build "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy wait-turn "$BUILD_ID" --project "$PROJECT_ID" --location "$LOCATION" --timeout 3600 + + # /workspace/plan.txt: "name version previous-commit - -", or empty when + # there is nothing to release (every step below then does nothing). + - id: Plan + name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim + entrypoint: bash + # The tool reads these from the environment: in a command, a run by hand's + # unresolved pusher ("$(push.pusher.name)") would run as a command. + env: + - BRANCH_NAME=$BRANCH_NAME + - PUSHER=${_PUSHER} + args: + - -c + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release-plan services/hackbot-ui --env "${_ENV}" --output /workspace/plan.txt - id: Build name: gcr.io/cloud-builders/docker @@ -34,20 +61,24 @@ steps: - -c - | set -euo pipefail + read -r _ version _ < /workspace/plan.txt || exit 0 + release="hackbot-ui@$$version" + sha="$COMMIT_SHA" + if [ "${_ENV}" = dev ]; then release="$$release+$${sha:0:12}"; fi image="${_AR_HOSTNAME}/${_AR_PROJECT_ID}/${_AR_REPOSITORY}/hackbot-ui:$COMMIT_SHA" docker build --no-cache -f services/hackbot-ui/Dockerfile \ --secret id=sentry_auth_token,env=SENTRY_AUTH_TOKEN \ --build-arg "SENTRY_DSN=${_SENTRY_DSN}" \ - --build-arg "SENTRY_RELEASE=hackbot-ui@$COMMIT_SHA" \ + --build-arg "SENTRY_RELEASE=$$release" \ + --label "org.opencontainers.image.version=$$version" \ --label "org.opencontainers.image.revision=$COMMIT_SHA" \ --label "org.opencontainers.image.created=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ -t "$$image" . docker push "$$image" - docker inspect --format='{{index .RepoDigests 0}}' "$$image" > /workspace/image.digest + docker inspect --format='{{index .RepoDigests 0}}' "$$image" > /workspace/hackbot-ui.digest - # Same digest to dev, then stage. Services have no shape gate: their - # infrastructure is written by hand in webservices-infra and changes there - # first. + # Services have no shape gate: their infrastructure is written by hand in + # webservices-infra and changes there first. - id: Deploy name: ghcr.io/astral-sh/uv:python3.14-bookworm-slim entrypoint: bash @@ -58,35 +89,41 @@ steps: - BUILD_ID=$BUILD_ID args: - -c - - | - set -euo pipefail - for env in dev stage; do - uv run --frozen --package hackbot-deploy python -m hackbot_deploy deploy services/hackbot-ui --env "$$env" --image "$$(cat /workspace/image.digest)" - done + - uv run --frozen --package hackbot-deploy python -m hackbot_deploy release --env "${_ENV}" --plan /workspace/plan.txt --digests /workspace # After the deploy, so a Sentry outage can't hold one up. - - id: Sentry release - name: getsentry/sentry-cli - secretEnv: - - SENTRY_AUTH_TOKEN - args: ["releases", "new", "hackbot-ui@$COMMIT_SHA", "--finalize"] - - id: Sentry commits + - id: Sentry name: getsentry/sentry-cli + entrypoint: sh secretEnv: - SENTRY_AUTH_TOKEN args: - [ - "releases", - "set-commits", - "hackbot-ui@$COMMIT_SHA", - "--commit", - "mozilla/bugbug@$COMMIT_SHA", - ] + - -c + - | + set -eu + read -r _ version previous _ < /workspace/plan.txt || exit 0 + release="hackbot-ui@$$version" + environment=staging + sha="$COMMIT_SHA" + if [ "${_ENV}" = dev ]; then + release="$$release+$$(echo "$$sha" | cut -c1-12)" + environment=development + fi + commits="mozilla/bugbug@$COMMIT_SHA" + if [ "$$previous" != - ]; then commits="mozilla/bugbug@$$previous..$COMMIT_SHA"; fi + sentry-cli releases new "$$release" + sentry-cli releases set-commits "$$release" --commit "$$commits" + sentry-cli releases finalize "$$release" + sentry-cli deploys new --release "$$release" -e "$$environment" -# The trigger sets it from the push event; empty when run any other way. +# _ENV is set by each trigger; _PUSHER from the push event, empty when run +# any other way. substitutions: + _ENV: stage _PUSHER: "" +timeout: 5400s + options: substitutionOption: MUST_MATCH logging: CLOUD_LOGGING_ONLY diff --git a/services/hackbot-ui/lib/environment.ts b/services/hackbot-ui/lib/environment.ts index 2e790eb7d7c..73dc6ccad6f 100644 --- a/services/hackbot-ui/lib/environment.ts +++ b/services/hackbot-ui/lib/environment.ts @@ -1,9 +1,18 @@ const PRODUCTION_HOSTNAME = "hackbot.moz.tools"; -const PRODUCTION_URL = `https://${PRODUCTION_HOSTNAME}`; +// The Sentry environment the Console runs in, from its hostname or URL. The +// names match the API's and the deploys the build pipeline records: +// production, staging (stage.), and development otherwise. export function resolveEnvironment(hostnameOrUrl = ""): string { - return hostnameOrUrl === PRODUCTION_HOSTNAME || - hostnameOrUrl.startsWith(PRODUCTION_URL) - ? "production" - : "development"; + let hostname = hostnameOrUrl; + if (hostnameOrUrl.includes("://")) { + try { + hostname = new URL(hostnameOrUrl).hostname; + } catch { + return "development"; + } + } + if (hostname === PRODUCTION_HOSTNAME) return "production"; + if (hostname.startsWith("stage.")) return "staging"; + return "development"; } diff --git a/tools/hackbot-deploy/README.md b/tools/hackbot-deploy/README.md index 14ff6f99a31..cec8c2abf56 100644 --- a/tools/hackbot-deploy/README.md +++ b/tools/hackbot-deploy/README.md @@ -85,8 +85,10 @@ in the same update, so the infrastructure and the image never disagree, not even in prod before a promotion. Apply the PR one environment at a time (dev, check, then stage, then prod) to control when each moves. -The manifest sync workflow fills them in once the build for a commit has -finished: +Pins are optional, and set by hand: a shape change without them keeps each +environment's running image (an expand fits it; for a contract, ship the code +that stops needing what's removed first). To move a contract and its image +together, pin the build's images when generating the infra PR: ```sh hackbot-deploy resolve-images bug-fix --tag \ @@ -138,9 +140,9 @@ What an agent deploy checks, in order: 3. **The gate.** A forward deploy needs that hash to equal the shape applied in the environment (`applied.json` in its registry bucket). A rollback needs it in `compatible`. -4. An older build doesn't replace a newer one (`--force` to override). A build - stops if an older build of its trigger is still running (`check-build`), but - a manual deploy can still race one, so this matters. +4. An older build doesn't replace a newer one (`--force` to override; dev + deploys always pass it). A build waits for older builds of its trigger + (`wait-turn`), but a manual deploy can still race one, so this matters. 5. The Job's containers match the images given, and its config-hash label agrees with `applied.json` (catching an incomplete apply). @@ -151,38 +153,70 @@ in which case Cloud Run keeps serving the old one. Then it updates the Job or service, reads it back, and writes `deployments/.json` (`deployments/agents/bug-fix.json`) with the images, -commit, build time and a -deployment history. The write is conditioned on the record's generation, so +version, commit, build time and a deployment history. The write is conditioned on the record's generation, so two concurrent deploys can't silently overwrite each other. A rollback steps back through that history one deployment at a time. -Exit codes: `0` done or nothing to do, `1` failed, `3` waiting for infra. A -pipeline should treat `3` as "stop here", not as a failure; -`agents/cloudbuild.yaml` does. +Exit codes: `0` done or nothing to do, `1` failed, `3` waiting for infra (the +image's shape isn't applied in that environment). The build configs treat `3` +as a failure: apply the infra PR, then rerun. The environment names (projects, the `-dev`/`-stage` suffix, bucket names) mirror the Terraform in webservices-infra. Change one, change both. ## Prod -Prod is never deployed by a build, and never without an approval. The -**Hackbot prod deploy** workflow offers every stage deploy to prod, one agent -or service at a time: once a build's check run succeeds, it compares stage with -prod (`promotion`), and if they differ, waits for a reviewer in the GitHub -environment for its kind: `hackbot-prod-agents` or `hackbot-prod-services`. On approval it runs `promote --expect`, which puts on exactly -the digests that were planned, or reports `superseded` if stage has moved on -(the newer build's run asks for its own approval). The approver is recorded as -`deployed_by`. - -From the Actions tab, `promote` does the same for one name, and `rollback` -steps prod back through its history in the `hackbot-prod-rollback` +Prod is never deployed by a build, and never without an approval. From the +Actions tab, the **Hackbot prod deploy** workflow's `promote` compares stage +with prod for one agent or service (`promotion`), and if they differ, waits for +a reviewer in the GitHub environment for its kind: `hackbot-prod-agents` or +`hackbot-prod-services`. On approval it runs `promote --expect`, which puts on +exactly the digests that were planned, or reports `superseded` if stage has +moved on. The approver is recorded as `deployed_by`. For a service, it also +marks its Sentry release (`@`) deployed to production, with +the `SENTRY_AUTH_TOKEN` secret of `hackbot-prod-services`. + +`rollback` steps prod back through its history in the `hackbot-prod-rollback` environment. The prod deploy account is bound to those three environments, so no workflow can deploy prod without passing their protection. A shape change reaches prod by applying the infra PR's prod directory instead, which puts on the images pinned for it. -## Building +## Building and releasing -`agents/cloudbuild.yaml` builds an agent's images with the hash label, and -`services//cloudbuild.yaml` builds the API and UI. Each then deploys -to dev and then stage. Their triggers are created in webservices-infra. +A release is a version bump: the `VERSION` file of an agent +(`agents//VERSION`) or a service (`services//VERSION`), which +its package also reads. Images are tagged with their commit and labelled with +their version (`org.opencontainers.image.version`). +Six Cloud Build triggers, created in webservices-infra, run the build configs: + +| trigger | runs on | deploys | +| --------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `stage-agents` | the default branch, touching `agents/*/VERSION` | to stage, every agent whose version isn't the one stage runs, one after another | +| `stage-hackbot-api`, `stage-hackbot-ui` | the default branch, touching `services//VERSION` | to stage, if its version isn't the one stage runs | +| `dev-agents` | `dev/agents/` or `dev/agents//` | that agent to dev, always | +| `dev-hackbot-api`, `dev-hackbot-ui` | `dev/services/` or `dev/services//` | the service to dev, always | + +```sh +git push -f upstream HEAD:dev/agents/bug-fix # try a branch in dev +git push -f upstream HEAD:dev/services/hackbot-api/my-fix # a suffix keeps branches apart +``` + +Dev is a shared sandbox: the last push wins. The branches have to be in +mozilla/bugbug, not a fork. + +A stage trigger run by hand deploys regardless of version: `stage-agents` +releases every agent, or just those in `_AGENTS=bug-fix,test-repair`, and a +service's trigger redeploys the service. A newer build waits for older builds +of the same trigger (`wait-turn`). + +The tool makes every decision; the build configs only build images and call +it. `release-plan` writes what to release (in dev, what the branch names; +otherwise each agent's, or the named service's, package version against the +environment's record, or everything on a run by hand), one `name version +previous-commit config-hash broker` line each. `release` deploys those whose +images built, one after another: one failing doesn't stop the others, and it +fails at the end, naming them; a rerun only redoes those. The services' builds +also make the Sentry release `@` (dev builds add +`+`), with the commits since what the environment ran before, and mark +its deploy there. diff --git a/tools/hackbot-deploy/hackbot_deploy/cli.py b/tools/hackbot-deploy/hackbot_deploy/cli.py index fa819bca0e1..05aa11dcd46 100644 --- a/tools/hackbot-deploy/hackbot_deploy/cli.py +++ b/tools/hackbot-deploy/hackbot_deploy/cli.py @@ -13,7 +13,7 @@ import typer -from hackbot_deploy import deploy, manifest +from hackbot_deploy import deploy, manifest, release from hackbot_deploy.gcp import GoogleCloud EXIT_WAITING_FOR_INFRA = 3 @@ -33,6 +33,7 @@ AgentsDir = Annotated[Path, typer.Option(help="Where the agents live.")] ServicesDir = Annotated[Path, typer.Option(help="Where the services live.")] +Root = Annotated[Path, typer.Option(help="The repository root.")] Name = Annotated[ str, typer.Argument(help="What to deploy: agents/ or services/."), @@ -217,19 +218,102 @@ def resolve_images( @app.command() -def check_build( +def wait_turn( build_id: str, project: Annotated[str, typer.Option()], location: Annotated[str, typer.Option()], + timeout: Annotated[ + int, typer.Option(help="Seconds to wait before failing.") + ] = 3600, ) -> None: - """Fail if an older build of this build's trigger is still running.""" - older = deploy.older_builds(GoogleCloud(), project, location, build_id) - if older: - print( - f"an older build of this trigger is still running ({', '.join(older)}); " - "rerun this trigger once it finishes to deploy this commit", + """Wait until no older build of this build's trigger is still running.""" + deploy.wait_for_older_builds( + GoogleCloud(), + project, + location, + build_id, + timeout_seconds=timeout, + on_wait=lambda older: print( + f"waiting for older builds of this trigger: {', '.join(older)}", file=sys.stderr, - ) + ), + ) + + +@app.command() +def release_plan( + env: EnvOption, + output: Annotated[Path, typer.Option(help="Where to write the plan.")], + names: Annotated[ + list[str] | None, + typer.Argument(help="What to consider (default: every agent with [deploy])."), + ] = None, + agents: Annotated[ + str, + typer.Option( + envvar="AGENTS", help="Comma-separated agents to consider instead." + ), + ] = "", + branch: Annotated[ + str, + typer.Option( + envvar="BRANCH_NAME", + help="The branch built: in dev, it names what to release.", + ), + ] = "", + pusher: Annotated[ + str, + typer.Option( + envvar="PUSHER", help="Who pushed; none means the build was run by hand." + ), + ] = "", + root: Root = Path("."), +) -> None: + """Write what a build releases into an environment (see release.py). + + dev releases what the branch names, always. Elsewhere, a push releases + what changed version; a build run by hand releases everything considered. + The build configs pass the branch, agents and pusher in the environment, + so none of them is ever part of a shell command. + """ + if agents: + names = [f"{deploy.AGENTS}/{a.strip()}" for a in agents.split(",") if a.strip()] + planned = release.plan( + GoogleCloud(), + deploy.ENVIRONMENTS[env], + root, + names=names, + branch=branch, + pusher=pusher, + ) + for item in planned: + print(f"release {item.name} {item.version}", file=sys.stderr) + if not planned: + print(f"nothing to release to {env}", file=sys.stderr) + output.write_text("".join(f"{item.line()}\n" for item in planned)) + + +@app.command("release") +def release_( + env: EnvOption, + plan: Annotated[Path, typer.Option(help="The plan release-plan wrote.")], + digests: Annotated[ + Path, typer.Option(help="Where the build wrote its .digest files.") + ], +) -> None: + """Deploy what the plan lists and the build built; fail naming what didn't make it.""" + planned = [release.Planned.parse(line) for line in plan.read_text().splitlines()] + cloud = GoogleCloud() + released, failed = release.run( + cloud, deploy.ENVIRONMENTS[env], planned, digests, actor=_actor(cloud) + ) + print(f"Released to {env}:") + for line in released or ["nothing"]: + print(f" {line}") + if failed: + print("Failed:") + for line in failed: + print(f" {line}") raise typer.Exit(1) diff --git a/tools/hackbot-deploy/hackbot_deploy/deploy.py b/tools/hackbot-deploy/hackbot_deploy/deploy.py index a3ac60d9087..dcf467ba9f1 100644 --- a/tools/hackbot-deploy/hackbot_deploy/deploy.py +++ b/tools/hackbot-deploy/hackbot_deploy/deploy.py @@ -28,6 +28,7 @@ import json import re +import time from collections.abc import Callable from dataclasses import dataclass, field from datetime import UTC, datetime @@ -37,6 +38,7 @@ LABEL_CONFIG_HASH = "hackbot.config_hash" LABEL_REVISION = "org.opencontainers.image.revision" LABEL_CREATED = "org.opencontainers.image.created" +LABEL_VERSION = "org.opencontainers.image.version" JOB_LABEL_CONFIG_HASH = "hackbot-config-hash" HISTORY_LIMIT = 10 # A registry image pinned by digest. The host is plain DNS labels: no "#", @@ -193,7 +195,14 @@ def _read_record( def _summary(record: dict[str, Any]) -> dict[str, Any]: return { k: record[k] - for k in ("images", "config_hash", "revision", "created", "deployed_at") + for k in ( + "images", + "config_hash", + "version", + "revision", + "created", + "deployed_at", + ) if k in record } @@ -276,6 +285,7 @@ def _ship( "runtime": runtime, "images": images, "config_hash": config_hash, + "version": labels.get(LABEL_VERSION), "revision": labels.get(LABEL_REVISION), "created": labels.get(LABEL_CREATED), "deployed_at": _now(), @@ -359,6 +369,7 @@ def _adopt_terraform_deploy( "runtime": applied["runtime"], "images": running, "config_hash": applied["applied"], + "version": labels.get(LABEL_VERSION), "revision": labels.get(LABEL_REVISION), "created": labels.get(LABEL_CREATED), "deployed_at": _now(), @@ -586,6 +597,7 @@ def promotion( "kind": kind, "environment": PROD_APPROVAL[kind], "images": record["images"], + "version": record.get("version"), "revision": record.get("revision"), "created": record.get("created"), "deployed_by": record.get("deployed_by"), @@ -625,6 +637,50 @@ def promote( # --- Builds -------------------------------------------------------------------- +def release_needed( + cloud: Cloud, env: Environment, deployable: str, version: str +) -> tuple[bool, str | None]: + """Say whether ``deployable`` at ``version`` is new to ``env``. + + Also returns the commit ``env`` runs now, the start of the release's + commit range. A release is a version bump: the same version again is + already there, whatever else changed. + """ + split_id(deployable) + record, _ = _read_record(cloud, env, deployable) + if record is None: + return True, None + return record.get("version") != version, record.get("revision") + + +def wait_for_older_builds( + cloud: Cloud, + project: str, + location: str, + build_id: str, + *, + timeout_seconds: float, + poll_seconds: float = 30.0, + on_wait: Callable[[list[str]], None] = lambda older: None, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, +) -> None: + """Return once no older build of this build's trigger is still going. + + Builds of one trigger deploy to one environment, so taking turns keeps + two of them from deploying at once while every build still deploys, in + the order they started. + """ + deadline = clock() + timeout_seconds + while older := older_builds(cloud, project, location, build_id): + if clock() >= deadline: + raise DeployError( + f"older builds of this trigger are still running after {timeout_seconds:.0f}s: {', '.join(older)}" + ) + on_wait(older) + sleep(poll_seconds) + + def built_images( cloud: Cloud, repository: str, @@ -650,13 +706,7 @@ def built_images( def older_builds(cloud: Cloud, project: str, location: str, build_id: str) -> list[str]: - """Return the builds of this build's trigger that started earlier and are still going. - - Deploys of one agent or service aren't serialised, and two at once can - leave it running one image while the record names the other. So a build - stops when an older one of its trigger is still going; rerunning the - trigger once that finishes deploys the newer commit. - """ + """Return the builds of this build's trigger that started earlier and are still going.""" build = cloud.get_build(project, location, build_id) trigger = build.get("buildTriggerId") if not trigger: # started by hand, not by a trigger diff --git a/tools/hackbot-deploy/hackbot_deploy/manifest.py b/tools/hackbot-deploy/hackbot_deploy/manifest.py index b39b78c25da..1a74650fb92 100644 --- a/tools/hackbot-deploy/hackbot_deploy/manifest.py +++ b/tools/hackbot-deploy/hackbot_deploy/manifest.py @@ -84,6 +84,22 @@ def load_config(path: Path) -> DeployConfig | None: raise ManifestError(f"{path}: invalid [deploy] table\n{exc}") from exc +def package_version(root: Path, deployable: str) -> str: + """Return a deployable's version: its VERSION file, which its package reads. + + Bumping it is what releases it. + """ + try: + version = (root / deployable / "VERSION").read_text().strip() + except OSError as exc: + raise ManifestError(f"{deployable}: no VERSION file ({exc})") from exc + if not version or any(c.isspace() for c in version): + raise ManifestError( + f"{deployable}: VERSION must hold one version, not {version!r}" + ) + return version + + def load_configs(agents_dir: Path) -> dict[str, DeployConfig]: """Return every deployable agent under ``agents_dir``, keyed by directory name.""" configs: dict[str, DeployConfig] = {} diff --git a/tools/hackbot-deploy/hackbot_deploy/release.py b/tools/hackbot-deploy/hackbot_deploy/release.py new file mode 100644 index 00000000000..a6f87d09f9e --- /dev/null +++ b/tools/hackbot-deploy/hackbot_deploy/release.py @@ -0,0 +1,156 @@ +"""Releases: what a build releases, and deploying what it built. + +A release is a version bump. A build plans which deployables to release into +its environment (``plan``), builds their images, then deploys them one after +another (``run``); one failing doesn't stop the rest. + +The plan is a text file the build config reads too, one deployable per line: +``name version previous-commit config-hash broker``, "-" for none. +""" + +import re +from dataclasses import dataclass +from pathlib import Path + +from hackbot_deploy import deploy, manifest + +# dev/agents/ or dev/services/, with an optional /. +_DEV_BRANCH = re.compile( + r"dev/(?Pagents|services)/(?P[a-z0-9][a-z0-9-]*)(/.+)?" +) + + +@dataclass(frozen=True) +class Planned: + name: str + version: str + previous: str | None = None + config_hash: str | None = None + has_broker: bool = False + + def line(self) -> str: + fields = [self.name, self.version, self.previous, self.config_hash] + return " ".join(f or "-" for f in fields) + ( + " broker" if self.has_broker else " -" + ) + + @classmethod + def parse(cls, line: str) -> "Planned": + name, version, previous, config_hash, broker = line.split() + return cls( + name, + version, + None if previous == "-" else previous, + None if config_hash == "-" else config_hash, + broker == "broker", + ) + + +def from_branch(branch: str) -> str: + """The deployable a dev branch names: dev/agents/bug-fix/x -> agents/bug-fix.""" + match = _DEV_BRANCH.fullmatch(branch) + if match is None: + raise deploy.DeployError( + f"{branch}: expected dev/agents/ or dev/services/, optionally /" + ) + return f"{match['kind']}/{match['name']}" + + +def by_hand(pusher: str) -> bool: + """Whether a build was started by hand: no pusher, or its binding unresolved.""" + return not pusher or pusher.startswith("$(") + + +def plan( + cloud: deploy.Cloud, + env: deploy.Environment, + root: Path, + *, + names: list[str] | None = None, + branch: str = "", + pusher: str = "", +) -> list[Planned]: + """Decide what to release into ``env``. + + dev releases what the branch names, always. Elsewhere, a push releases + what changed version among ``names`` (default: every agent with a + [deploy] table); a build started by hand releases them regardless. + """ + if env.name == "dev": + candidates, force = [from_branch(branch)], True + else: + candidates = names or [ + f"{deploy.AGENTS}/{agent}" + for agent in sorted(manifest.load_configs(root / deploy.AGENTS)) + ] + force = by_hand(pusher) + + planned = [] + for name in candidates: + kind, short = deploy.split_id(name) + version = manifest.package_version(root, name) + needed, previous = deploy.release_needed(cloud, env, name, version) + if not (needed or force): + continue + config_hash, has_broker = None, False + if kind == deploy.AGENTS: + config = manifest.load_config(root / name / "hackbot.toml") + if config is None: + raise deploy.DeployError(f"{name} has no [deploy] table") + entry = config.manifest_entry() + config_hash, has_broker = ( + manifest.config_hash(short, entry), + "broker" in entry, + ) + planned.append(Planned(name, version, previous, config_hash, has_broker)) + return planned + + +def built_images(planned: Planned, digests: Path) -> dict[str, str] | None: + """The digests the build wrote for ``planned``, or None if it didn't build. + + The build writes -agent.digest and -broker.digest for an + agent, .digest for a service. + """ + kind, short = deploy.split_id(planned.name) + if kind == deploy.SERVICES: + files = {short: digests / f"{short}.digest"} + else: + containers = ["agent", "broker"] if planned.has_broker else ["agent"] + files = {c: digests / f"{short}-{c}.digest" for c in containers} + if not all(path.exists() for path in files.values()): + return None + return {container: path.read_text().strip() for container, path in files.items()} + + +def run( + cloud: deploy.Cloud, + env: deploy.Environment, + planned: list[Planned], + digests: Path, + *, + actor: str, +) -> tuple[list[str], list[str]]: + """Deploy every planned deployable that built. Return what was released and what failed.""" + released, failed = [], [] + for item in planned: + kind, short = deploy.split_id(item.name) + images = built_images(item, digests) + if images is None: + failed.append(f"{item.name}: its images didn't build") + continue + common = {"force": env.name == "dev", "actor": actor} + try: + if kind == deploy.SERVICES: + deploy.deploy_service(cloud, short, env, images[short], **common) + else: + deploy.deploy_agent(cloud, short, env, images, **common) + except deploy.WaitingForInfra: + failed.append( + f"{item.name}: its shape isn't applied in {env.name}; apply the infra PR, then rerun" + ) + except deploy.DeployError as exc: + failed.append(f"{item.name}: {exc}") + else: + released.append(f"{item.name} {item.version}") + return released, failed diff --git a/tools/hackbot-deploy/tests/test_deploy.py b/tools/hackbot-deploy/tests/test_deploy.py index e14c5257347..a73e03dfab2 100644 --- a/tools/hackbot-deploy/tests/test_deploy.py +++ b/tools/hackbot-deploy/tests/test_deploy.py @@ -2,7 +2,7 @@ from typing import Any import pytest -from hackbot_deploy import cli, deploy +from hackbot_deploy import cli, deploy, release from hackbot_deploy.deploy import ( ENVIRONMENTS, DeployError, @@ -77,12 +77,17 @@ def service(self, env: deploy.Environment, service: str) -> None: } def build( - self, ref: str, config_hash: str, created: str = "2026-10-09T10:00:00Z" + self, + ref: str, + config_hash: str, + created: str = "2026-10-09T10:00:00Z", + version: str = "1.0.0", ) -> str: self.labels[ref] = { deploy.LABEL_CONFIG_HASH: config_hash, deploy.LABEL_CREATED: created, deploy.LABEL_REVISION: "abc123", + deploy.LABEL_VERSION: version, } return ref @@ -789,3 +794,197 @@ def test_a_service_promotion_waits_in_the_services_environment( planned = deploy.promotion(services, "services/hackbot-api", DEV, STAGE) assert planned is not None assert planned["environment"] == "hackbot-prod-services" + + +def test_the_record_carries_the_version(cloud: FakeCloud) -> None: + ship(cloud, 1) + assert cloud.record(DEV, "agents/bug-fix")["version"] == "1.0.0" + + +def test_a_release_is_a_version_new_to_the_environment(cloud: FakeCloud) -> None: + assert deploy.release_needed(cloud, DEV, "agents/bug-fix", "1.0.0") == (True, None) + ship(cloud, 1) + assert deploy.release_needed(cloud, DEV, "agents/bug-fix", "1.0.0") == ( + False, + "abc123", + ) + assert deploy.release_needed(cloud, DEV, "agents/bug-fix", "1.1.0") == ( + True, + "abc123", + ) + + +def test_a_build_waits_for_older_builds_of_its_trigger() -> None: + fake = FakeCloud() + fake.builds = [ + build("older", "2026-10-10T13:00:00Z"), + build("this", "2026-10-10T13:25:40Z"), + ] + waited: list[list[str]] = [] + + def older_finishes(seconds: float) -> None: + fake.builds[0]["status"] = "SUCCESS" + + deploy.wait_for_older_builds( + fake, + "p", + "us-west1", + "this", + timeout_seconds=60, + on_wait=waited.append, + sleep=older_finishes, + ) + assert waited == [["older"]] + + +def test_waiting_gives_up_after_the_timeout() -> None: + fake = FakeCloud() + fake.builds = [ + build("older", "2026-10-10T13:00:00Z"), + build("this", "2026-10-10T13:25:40Z"), + ] + now = iter([0.0, 30.0, 61.0]) + with pytest.raises(DeployError, match="still running after 60s"): + deploy.wait_for_older_builds( + fake, + "p", + "us-west1", + "this", + timeout_seconds=60, + sleep=lambda seconds: None, + clock=lambda: next(now), + ) + + +def write_agent(root, agent: str, version: str, broker: bool = False) -> None: + (root / "agents" / agent).mkdir(parents=True) + (root / "agents" / agent / "VERSION").write_text(f"{version}\n") + table = '[deploy]\nruntime = "cloud_run_job"\ncpu = "1"\nmemory = "1Gi"\n' + if broker: + table += "[deploy.broker]\n" + (root / "agents" / agent / "hackbot.toml").write_text(table) + + +@pytest.fixture +def repo(tmp_path, cloud: FakeCloud): + """bug-fix 1.0.0 already in stage, test-repair 2.0.0 never deployed.""" + write_agent(tmp_path, "bug-fix", "1.0.0", broker=True) + write_agent(tmp_path, "test-repair", "2.0.0") + cloud.apply(STAGE, "bug-fix", "h2") + cloud.job(STAGE, "bug-fix", ["agent", "broker"], label="h2") + deploy_agent( + cloud, + "bug-fix", + STAGE, + { + "agent": cloud.build(image("bug-fix-agent", 1), "h2"), + "broker": cloud.build(image("bug-fix-broker", 1), "h2"), + }, + ) + return tmp_path + + +def test_a_push_releases_what_changed_version(cloud: FakeCloud, repo) -> None: + planned = release.plan(cloud, STAGE, repo, pusher="marco-c") + assert [(p.name, p.version, p.previous) for p in planned] == [ + ("agents/test-repair", "2.0.0", None) + ] + assert planned[0].config_hash and not planned[0].has_broker + + +@pytest.mark.parametrize("pusher", ["", "$(push.pusher.name)"]) +def test_a_build_run_by_hand_releases_regardless( + cloud: FakeCloud, repo, pusher: str +) -> None: + planned = release.plan(cloud, STAGE, repo, names=["agents/bug-fix"], pusher=pusher) + assert [(p.name, p.previous, p.has_broker) for p in planned] == [ + ("agents/bug-fix", "abc123", True) + ] + + +@pytest.mark.parametrize( + ("branch", "expected"), + [ + ("dev/agents/bug-fix", "agents/bug-fix"), + ("dev/agents/bug-fix/suhaib-retry", "agents/bug-fix"), + ("dev/services/hackbot-api/x/y", "services/hackbot-api"), + ], +) +def test_dev_branches_name_what_to_release(branch: str, expected: str) -> None: + assert release.from_branch(branch) == expected + + +@pytest.mark.parametrize( + "branch", ["master", "dev/agents/", "dev/agents/$(rm)", "dev/other/bug-fix"] +) +def test_other_branches_are_refused(branch: str) -> None: + with pytest.raises(DeployError, match="expected dev/agents"): + release.from_branch(branch) + + +def test_dev_releases_the_branch_always(cloud: FakeCloud, repo) -> None: + planned = release.plan( + cloud, DEV, repo, branch="dev/agents/bug-fix/try", pusher="x" + ) + assert [p.name for p in planned] == ["agents/bug-fix"] + + +def test_plan_lines_round_trip() -> None: + item = release.Planned("agents/bug-fix", "1.0.0", None, "h2", True) + assert item.line() == "agents/bug-fix 1.0.0 - h2 broker" + assert release.Planned.parse(item.line()) == item + service = release.Planned("services/hackbot-api", "0.2.0", "abc") + assert release.Planned.parse(service.line()) == service + + +def test_release_deploys_what_built_and_names_what_failed( + cloud: FakeCloud, tmp_path +) -> None: + (tmp_path / "bug-fix-agent.digest").write_text( + cloud.build(image("bug-fix-agent", 2), "h2") + "\n" + ) + (tmp_path / "bug-fix-broker.digest").write_text( + cloud.build(image("bug-fix-broker", 2), "h2") + "\n" + ) + # Dev hasn't applied test-repair's shape. + (tmp_path / "test-repair-agent.digest").write_text( + cloud.build(image("test-repair-agent", 2), "h1") + "\n" + ) + planned = [ + release.Planned("agents/bug-fix", "1.1.0", None, "h2", True), + release.Planned("agents/test-repair", "2.1.0", None, "h1", False), + release.Planned("agents/build-repair", "3.0.0", None, "h3", False), + ] + released, failed = release.run(cloud, DEV, planned, tmp_path, actor="test") + assert released == ["agents/bug-fix 1.1.0"] + assert failed == [ + "agents/test-repair: its shape isn't applied in dev; apply the infra PR, then rerun", + "agents/build-repair: its images didn't build", + ] + assert cloud.record(DEV, "agents/bug-fix")["images"]["agent"] == image( + "bug-fix-agent", 2 + ) + + +def test_release_plan_reads_the_build_environment( + cloud: FakeCloud, repo, tmp_path, monkeypatch +) -> None: + monkeypatch.setattr(cli, "GoogleCloud", lambda: cloud) + monkeypatch.setenv("AGENTS", "bug-fix") + monkeypatch.setenv("PUSHER", "$(push.pusher.name)") # a run by hand + monkeypatch.setenv("BRANCH_NAME", "master") + plan = tmp_path / "plan.txt" + with pytest.raises(SystemExit) as exit_: + cli.app( + [ + "release-plan", + "--env", + "stage", + "--output", + str(plan), + "--root", + str(repo), + ] + ) + assert exit_.value.code == 0 + assert plan.read_text().split()[:3] == ["agents/bug-fix", "1.0.0", "abc123"] diff --git a/tools/hackbot-deploy/tests/test_manifest.py b/tools/hackbot-deploy/tests/test_manifest.py index dc327b2108f..94744e8e2a2 100644 --- a/tools/hackbot-deploy/tests/test_manifest.py +++ b/tools/hackbot-deploy/tests/test_manifest.py @@ -275,3 +275,19 @@ def _regenerate_with( ) -> tuple[dict, Change]: generated, changes = build_manifest({"a": cfg}, previous, **kw) # type: ignore[arg-type] return generated["agents"]["a"], changes["a"] + + +def test_package_versions(tmp_path: Path) -> None: + for name, content in (("agents/a", "0.3.1\n"), ("services/ui", "1.2.0")): + (tmp_path / name).mkdir(parents=True) + (tmp_path / name / "VERSION").write_text(content) + (tmp_path / "services" / "none").mkdir(parents=True) + (tmp_path / "services" / "two").mkdir(parents=True) + (tmp_path / "services" / "two" / "VERSION").write_text("1.0\n2.0\n") + + assert manifest.package_version(tmp_path, "agents/a") == "0.3.1" + assert manifest.package_version(tmp_path, "services/ui") == "1.2.0" + with pytest.raises(manifest.ManifestError, match="no VERSION file"): + manifest.package_version(tmp_path, "services/none") + with pytest.raises(manifest.ManifestError, match="one version"): + manifest.package_version(tmp_path, "services/two")