From e441e513dde4d72ccc9972fdd5e4da4310199483 Mon Sep 17 00:00:00 2001 From: Yury Kovalev Date: Fri, 6 Nov 2026 12:09:09 +0100 Subject: [PATCH 1/3] chore: add e2e tests for CRC --- .github/workflows/e2e-tests-crc.yml | 107 ++++++++++++++++++++++++++++ .github/workflows/main-push.yml | 7 ++ .github/workflows/pr.yml | 8 +++ 3 files changed, 122 insertions(+) create mode 100644 .github/workflows/e2e-tests-crc.yml diff --git a/.github/workflows/e2e-tests-crc.yml b/.github/workflows/e2e-tests-crc.yml new file mode 100644 index 00000000..609eeadf --- /dev/null +++ b/.github/workflows/e2e-tests-crc.yml @@ -0,0 +1,107 @@ +name: E2E Tests (CRC OpenShift) + +on: + workflow_call: + inputs: + image: + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + QUAY_RHACS_ENG_RO_USERNAME: + required: true + QUAY_RHACS_ENG_RO_PASSWORD: + required: true + +env: + REGISTRY: quay.io + +jobs: + e2e-tests-crc: + # CRC requires nested virtualization and enough RAM for the OpenShift VM. + runs-on: ubuntu-24.04 + timeout-minutes: 180 + defaults: + run: + shell: bash + env: + SKIP_OLM_TESTS: "true" + ROXIE_RESOURCE_PROFILE: "tiny" + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Free disk space for CRC + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + # Keep swap available while OpenShift and the Go tests share the runner. + swap-storage: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Log in to Quay.io + uses: docker/login-action@v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract roxie binary from image + env: + ROXIE_IMAGE: ${{ inputs.image }} + run: | + docker create --name roxie-extract "$ROXIE_IMAGE" + docker cp roxie-extract:/usr/local/bin/roxie "$GITHUB_WORKSPACE/roxie" + docker rm roxie-extract + + - name: Install roxie binary + run: | + sudo install -m 0755 "$GITHUB_WORKSPACE/roxie" /usr/local/bin/roxie + roxie version + + - name: Create CRC OpenShift cluster + uses: crc-org/crc-github-action@v1.0.1 + with: + preset: openshift + cpus: 4 + memory: 10752 + disk: 40 + + - name: Wait for cluster readiness + run: | + kubectl wait --for=condition=Ready nodes --all --timeout=10m + kubectl get namespaces + + - name: Run e2e tests + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: make run-test-e2e 2>&1 | tee e2e-crc.log + + - name: Collect cluster diagnostics + if: failure() + continue-on-error: true + run: | + crc status || true + kubectl get nodes -o wide || true + kubectl get pods -A -o wide || true + kubectl get events -A --sort-by=.lastTimestamp || true + + - name: Upload e2e log + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-crc-log + path: e2e-crc.log + if-no-files-found: ignore diff --git a/.github/workflows/main-push.yml b/.github/workflows/main-push.yml index 9beecbab..29a5a6b2 100644 --- a/.github/workflows/main-push.yml +++ b/.github/workflows/main-push.yml @@ -42,6 +42,13 @@ jobs: image: ${{ needs.build-roxie-image.outputs.image }} secrets: inherit + e2e-tests-crc: + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-crc.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-dev-cluster: if: ${{ always() && needs.create-dev-cluster.result == 'success' }} needs: [ create-dev-cluster, e2e-tests ] diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8a0a2e1d..5dc71cb1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -85,6 +85,14 @@ jobs: image: ${{ needs.build-roxie-image.outputs.image }} secrets: inherit + e2e-tests-crc: + if: github.event.pull_request.head.repo.full_name == github.repository + needs: [ build-roxie-image ] + uses: ./.github/workflows/e2e-tests-crc.yml + with: + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + delete-gke-cluster: if: ${{ always() && needs.create-gke-cluster.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository }} needs: [ create-gke-cluster, e2e-tests ] From cd43a10fd964bf99ee156ba508a38145096a1e07 Mon Sep 17 00:00:00 2001 From: Yury Kovalev Date: Fri, 6 Nov 2026 15:26:50 +0100 Subject: [PATCH 2/3] drop default --- .github/workflows/e2e-tests-crc.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/e2e-tests-crc.yml b/.github/workflows/e2e-tests-crc.yml index 609eeadf..959e7803 100644 --- a/.github/workflows/e2e-tests-crc.yml +++ b/.github/workflows/e2e-tests-crc.yml @@ -74,9 +74,6 @@ jobs: uses: crc-org/crc-github-action@v1.0.1 with: preset: openshift - cpus: 4 - memory: 10752 - disk: 40 - name: Wait for cluster readiness run: | From b006dec4a55c6f3c1db7514641b9e07227ca0ecb Mon Sep 17 00:00:00 2001 From: Yury Kovalev Date: Mon, 9 Nov 2026 12:17:16 +0100 Subject: [PATCH 3/3] Reclaim some memory for CRC e2e tests --- .github/scripts/configure-crc-e2e.sh | 127 +++++++++++++++++++++++++++ .github/workflows/e2e-tests-crc.yml | 15 +++- 2 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/configure-crc-e2e.sh diff --git a/.github/scripts/configure-crc-e2e.sh b/.github/scripts/configure-crc-e2e.sh new file mode 100644 index 00000000..8ddef12d --- /dev/null +++ b/.github/scripts/configure-crc-e2e.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Only for the disposable CRC cluster created by e2e-tests-crc.yml. Do not run +# against a developer's CRC: CVO overrides disable normal operator management. +if [[ "${GITHUB_ACTIONS:-}" != "true" || "${SKIP_OLM_TESTS:-}" != "true" ]]; then + echo "This script requires GitHub Actions and SKIP_OLM_TESTS=true" >&2 + exit 1 +fi +server=$(oc config view --minify -o jsonpath='{.clusters[0].cluster.server}') +if [[ "$server" != "https://api.crc.testing:6443" ]]; then + echo "Refusing to configure a non-CRC cluster: $server" >&2 + exit 1 +fi + +wait_for_empty() { + local namespace=$1 resource=$2 + shift 2 + local resources deadline=$((SECONDS + 300)) + while true; do + # CRC's API can briefly reset connections while the cluster reconciles. + # A failed list is not an empty list: retry it within the same deadline. + if resources=$(oc --request-timeout=15s get "$resource" -n "$namespace" "$@" -o name); then + if [[ -z "$resources" ]]; then + return + fi + else + resources="API request failed" + echo "Retrying list of $resource in $namespace after an API error" >&2 + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for $resource in $namespace: $resources" >&2 + return 1 + fi + sleep 5 + done +} + +echo "Node memory allocation before trimming OpenShift" +oc describe nodes + +# Use the supported removal settings first so their operators clean up their +# operands. ACS pulls from Quay, and these tests never use the OpenShift UI, +# internal registry, sample builds, or OLM. Do not alter ACS/Scanner V4 settings. +# https://docs.okd.io/4.22/web_console/disabling-web-console.html +oc patch consoles.operator.openshift.io cluster --type=merge \ + --patch='{"spec":{"managementState":"Removed"}}' +oc patch configs.imageregistry.operator.openshift.io cluster --type=merge \ + --patch='{"spec":{"managementState":"Removed"}}' +oc patch configs.samples.operator.openshift.io cluster --type=merge \ + --patch='{"spec":{"managementState":"Removed"}}' +oc patch operatorhub.config.openshift.io cluster --type=merge \ + --patch='{"spec":{"disableAllDefaultSources":true}}' +oc patch networks.operator.openshift.io cluster --type=merge \ + --patch='{"spec":{"disableNetworkDiagnostics":true}}' + +wait_for_empty openshift-console deployments +wait_for_empty openshift-console pods +wait_for_empty openshift-image-registry deployments image-registry --ignore-not-found +wait_for_empty openshift-image-registry pods -l docker-registry=default +wait_for_empty openshift-marketplace catalogsources +wait_for_empty openshift-marketplace pods -l olm.catalogSource +wait_for_empty openshift-network-diagnostics pods +oc wait configs.samples.operator.openshift.io/cluster \ + --for=jsonpath='{.status.managementState}'=Removed --timeout=5m + +# Wait for the migrator operator to stabilize before stopping its controllers. +# This CI cluster is never upgraded and its fixed CRC VM needs no Machine API +# provisioning. Keep the machine approver and machine-config stack running. +oc wait clusteroperator/kube-storage-version-migrator \ + --for=condition=Available=True --timeout=5m +oc wait clusteroperator/kube-storage-version-migrator \ + --for=condition=Progressing=False --timeout=5m + +# Enabled capabilities cannot be disabled after installation. For this +# short-lived cluster only, prevent CVO from restoring unused operators, using +# the same per-object override mechanism CRC uses to disable monitoring/cloud +# operators. Preserve all existing CRC overrides. Core cluster operators remain +# managed; we do not scale CVO itself down. +deployments='[ + ["openshift-operator-lifecycle-manager", "olm-operator"], + ["openshift-operator-lifecycle-manager", "catalog-operator"], + ["openshift-operator-lifecycle-manager", "package-server-manager"], + ["openshift-operator-lifecycle-manager", "packageserver"], + ["openshift-marketplace", "marketplace-operator"], + ["openshift-console-operator", "console-operator"], + ["openshift-cluster-samples-operator", "cluster-samples-operator"], + ["openshift-image-registry", "cluster-image-registry-operator"], + ["openshift-kube-storage-version-migrator-operator", "kube-storage-version-migrator-operator"], + ["openshift-kube-storage-version-migrator", "migrator"], + ["openshift-machine-api", "machine-api-operator"], + ["openshift-machine-api", "control-plane-machine-set-operator"] +]' +patch=$(oc get clusterversion version -o json | jq --argjson deployments "$deployments" ' + . as $version + | + ($deployments | map({group: "apps", kind: "Deployment", namespace: .[0], name: .[1], unmanaged: true})) + + [{group: "apiregistration.k8s.io", kind: "APIService", namespace: "", name: "v1.packages.operators.coreos.com", unmanaged: true}] + | . as $disabled + | reduce $disabled[] as $item (($version.spec.overrides // []); + map(select([.group, .kind, .namespace, .name] != [$item.group, $item.kind, $item.namespace, $item.name])) + [$item]) + | {spec: {overrides: .}} +') +oc patch clusterversion version --type=merge --patch="$patch" +generation=$(oc get clusterversion version -o jsonpath='{.metadata.generation}') +oc wait clusterversion/version \ + --for="jsonpath={.status.observedGeneration}=$generation" --timeout=5m + +# Stop OLM before its managed packageserver, then stop the remaining unused +# operators. Wait for each deployment to terminate before stopping its children. +while IFS=$'\t' read -r namespace name; do + if [[ "$namespace" == "openshift-operator-lifecycle-manager" && "$name" == "packageserver" ]]; then + # Its managers have stopped. Remove the registration before stopping the + # server so discovery/namespace deletion never depend on a dead endpoint. + oc delete apiservice v1.packages.operators.coreos.com --ignore-not-found + fi + oc scale deployment "$name" -n "$namespace" --replicas=0 + selector=$(oc get deployment "$name" -n "$namespace" -o json | jq -er ' + .spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",") + | if length == 0 then error("Deployment has no label selector") else . end + ') + wait_for_empty "$namespace" pods -l "$selector" +done < <(jq -r '.[] | @tsv' <<< "$deployments") + +echo "Node memory allocation after trimming OpenShift" +oc describe nodes +oc get --raw=/readyz diff --git a/.github/workflows/e2e-tests-crc.yml b/.github/workflows/e2e-tests-crc.yml index 959e7803..3a441603 100644 --- a/.github/workflows/e2e-tests-crc.yml +++ b/.github/workflows/e2e-tests-crc.yml @@ -21,7 +21,6 @@ env: jobs: e2e-tests-crc: - # CRC requires nested virtualization and enough RAM for the OpenShift VM. runs-on: ubuntu-24.04 timeout-minutes: 180 defaults: @@ -74,23 +73,33 @@ jobs: uses: crc-org/crc-github-action@v1.0.1 with: preset: openshift + cpus: 4 + # Leave roughly 4 GiB for the host and Go tests on the 16 GB runner. + memory: 12288 - name: Wait for cluster readiness run: | kubectl wait --for=condition=Ready nodes --all --timeout=10m kubectl get namespaces - - name: Run e2e tests + # Reclaim memory to avoid insufficient memory errors + - name: Disable unused OpenShift services + run: bash .github/scripts/configure-crc-e2e.sh + + - name: Run happy-path smoke test env: REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - run: make run-test-e2e 2>&1 | tee e2e-crc.log + run: | + go test -v -tags=e2e -run '^TestDeployBothSimple$' \ + -timeout=90m -parallel=1 ./tests/e2e/... 2>&1 | tee e2e-crc.log - name: Collect cluster diagnostics if: failure() continue-on-error: true run: | crc status || true + kubectl describe nodes || true kubectl get nodes -o wide || true kubectl get pods -A -o wide || true kubectl get events -A --sort-by=.lastTimestamp || true