Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 50 additions & 30 deletions .github/workflows/basic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,42 @@ name: Basic tests

on: [push, pull_request]

# A new push to the same branch/PR cancels the previous run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
pre-commit:
name: pre-commit
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# keep 3.11 until Renovate updates the pinned hooks (flynt 0.77 and
# pyupgrade v3.3.1 predate the removal of ast.Str in Python 3.12)
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.11'
- name: Install pre-commit
run: pip install pre-commit
- name: Run pre-commit
run: pre-commit run --all-files --show-diff-on-failure
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
- uses: pre-commit-ci/lite-action@062bca0919bc9d6e66755cc05074b70c77e111fc # v1.2.0
if: always()

shellcheck:
name: shellcheck
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
timeout-minutes: 30

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Run shellcheck
# TODO This should cover more than just tests/CI
# Excluded codes related to sourcing files
Expand All @@ -32,13 +47,16 @@ jobs:
find tests/CI -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source

pycodestyle:
name: pycodestyle
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
timeout-minutes: 30

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.11'
- name: Installing dependencies
Expand All @@ -54,13 +72,10 @@ jobs:
fi

check:
name: check (${{ matrix.command }})
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
timeout-minutes: 30
defaults:
# Activate the conda environment automatically in each step
run:
shell: bash -l {0}

strategy:
fail-fast: False
Expand All @@ -74,28 +89,30 @@ jobs:
- mypy

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Fail-fast for outdated pipelines
run: .github/workflows/fail-fast.sh
- uses: conda-incubator/setup-miniconda@v3
- uses: prefix-dev/setup-pixi@9936258fee697aff89b3fed6ca1f80a382ef694e # v0.11.3
with:
environment-file: environment.yml
cache: true
- name: Run tests
run: |
# FIXME: The unit tests currently only work with editable installs
pip install -e .[server,testing]
${{ matrix.command }}
pixi run python -m pip install -e .[server,testing]
# eval is needed to re-parse the shell quoting (e.g. -k 'not A and not B')
# which is lost by the plain $COMMAND expansion
eval "pixi run $COMMAND"
env:
REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}
COMMAND: ${{ matrix.command }}

check-py314:
name: check-py314 (${{ matrix.command }})
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
timeout-minutes: 30
defaults:
# Activate the conda environment automatically in each step
run:
shell: bash -l {0}

strategy:
fail-fast: False
Expand All @@ -108,19 +125,22 @@ jobs:
- pylint -j 0 -E src/

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Fail-fast for outdated pipelines
run: .github/workflows/fail-fast.sh
- name: fix python3.14
run: |
sed -i "s/python =3.11/python =3.14/g" environment.yml
- uses: conda-incubator/setup-miniconda@v3
- uses: prefix-dev/setup-pixi@9936258fee697aff89b3fed6ca1f80a382ef694e # v0.11.3
with:
environment-file: environment.yml
cache: true
environments: py314
- name: Run tests
run: |
# FIXME: The unit tests currently only work with editable installs
pip install -e .[server,testing]
${{ matrix.command }}
pixi run -e py314 python -m pip install -e .[server,testing]
# eval is needed to re-parse the shell quoting (e.g. -k 'not A and not B')
# which is lost by the plain $COMMAND expansion
eval "pixi run -e py314 $COMMAND"
env:
REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}
COMMAND: ${{ matrix.command }}
56 changes: 40 additions & 16 deletions .github/workflows/deployment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ on:
version:
description: "Override the release version number (e.g. 8.0.0a5)"

# Don't cancel: a queued run may be a release waiting to publish.
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false

permissions:
contents: read

jobs:
deploy-pypi:
name: PyPI deployment
Expand All @@ -27,14 +35,17 @@ jobs:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# The PAT is required to push the release notes commit to the protected
# release branches, and its persisted credentials are used for the push.
# zizmor: ignore[secrets-outside-env,artipacked] PAT required to push to the protected release branches
token: ${{ secrets.PAT || github.token }}
- run: |
git fetch --prune --unshallow
git config --global user.email "ci@diracgrid.org"
git config --global user.name "DIRACGrid CI"
- uses: actions/setup-python@v5
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.11'
- name: Installing dependencies
Expand All @@ -50,30 +61,38 @@ jobs:
run: |
python -m readme_renderer README.rst -o /tmp/README.html
- name: Prepare release notes
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
run: |
set -xeuo pipefail
IFS=$'\n\t'
PREV_VERSION=$(git describe --tags --abbrev=0 --match '*[0-9].[0-9]*' --exclude 'v[0-9]r*' --exclude 'v[0-9][0-9]r*')
REFERENCE_BRANCH=${GITHUB_REF#refs/heads/}
echo "Making release notes for ${REFERENCE_BRANCH} since ${PREV_VERSION}"
./docs/diracdoctools/scripts/dirac-docs-get-release-notes.py \
--token "${{ secrets.GITHUB_TOKEN }}" \
--token "${GITHUB_TOKEN}" \
--sinceTag "${PREV_VERSION}" \
--branches "${REFERENCE_BRANCH}" \
--repo "${{ github.repository }}" \
--repo "${GH_REPO}" \
> release.notes.new
cat release.notes.new
- name: Create tag if required
id: check-tag
env:
REPO: ${{ github.repository }}
EVENT_NAME: ${{ github.event_name }}
EVENT_REF: ${{ github.event.ref }}
VERSION_INPUT: ${{ github.event.inputs.version }}
run: |
set -xeuo pipefail
IFS=$'\n\t'
# Only do a real release for workflow_dispatch events from DIRACGrid/DIRAC for integration for Python 3 compatible branches
if [[ "${{ github.repository }}" == "DIRACGrid/DIRAC" ]]; then
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)r[0-9]+)$ ]]; then
if [[ "${REPO}" == "DIRACGrid/DIRAC" ]]; then
if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then
if [[ "${EVENT_REF}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)r[0-9]+)$ ]]; then
echo "Will create a real release"
export NEW_VERSION="v${{ github.event.inputs.version }}"
export NEW_VERSION="v${VERSION_INPUT}"
if [[ "${NEW_VERSION}" == "v" ]]; then
# If version wasn't given as an input to the workflow, use setuptools_scm to guess while removing "dev" portion of the version number
NEW_VERSION=v$(python -m setuptools_scm | sed 's@Guessed Version @@g' | sed -E 's@(\.dev|\+g).+@@g')
Expand All @@ -83,12 +102,12 @@ jobs:
# Validate the version
# Ensure the version doesn't look like a PEP-440 "dev release" (which might happen if the automatic version bump has issues)
python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif v.is_devrelease:\n raise ValueError(v)'
if [[ "${{ github.event.ref }}" =~ ^refs/heads/integration$ ]]; then
if [[ "${EVENT_REF}" =~ ^refs/heads/integration$ ]]; then
# If we're releasing from integration it must be a pre-release
python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif not v.is_prerelease:\n raise ValueError("integration should only be used for pre-releases")'
elif [[ "${{ github.event.ref }}" != "$(python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nprint(f"refs/heads/rel-v{v.major}r{v.minor}")')" ]]; then
elif [[ "${EVENT_REF}" != "$(python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nprint(f"refs/heads/rel-v{v.major}r{v.minor}")')" ]]; then
# If we're not releasing from integration the version should match the rel-vXrY branch name
echo "$NEW_VERSION is an invalid version for ${{ github.event.ref }}"
echo "$NEW_VERSION is an invalid version for ${EVENT_REF}"
exit 2
fi
# Commit the release notes
Expand Down Expand Up @@ -123,15 +142,18 @@ jobs:
retention-days: 5
- name: Make release on GitHub
if: steps.check-tag.outputs.create-release == 'true'
env:
NEW_VERSION: ${{ steps.check-tag.outputs.new-version }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
run: |
set -e
export NEW_VERSION=${{ steps.check-tag.outputs.new-version }}
echo "Pushing tagged release notes to ${GITHUB_REF#refs/heads/}"
git push "origin" "${GITHUB_REF#refs/heads/}"
echo "Making GitHub release for ${NEW_VERSION}"
.github/workflows/make_release.py \
--repo="${{ github.repository }}" \
--token="${{ secrets.GITHUB_TOKEN }}" \
--repo="${GH_REPO}" \
--token="${GITHUB_TOKEN}" \
--version="${NEW_VERSION}" \
--rev="$(git rev-parse HEAD)" \
--release-notes-fn="release.notes.new"
Expand All @@ -157,6 +179,7 @@ jobs:
uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1

deploy_CVMFS:
name: deploy_CVMFS
runs-on: "ubuntu-latest"
# Wait for the PyPI upload: sync_packages.sh must see the new release.
# publish is skipped when no release is made; only skip CVMFS if it failed.
Expand All @@ -166,7 +189,7 @@ jobs:
# conda is no longer preinstalled (nor on the PATH) of the ubuntu-latest
# runner images; install it explicitly since the next steps rely on it
- name: Install conda
uses: conda-incubator/setup-miniconda@v3
uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3
with:
miniforge-version: latest
conda-remove-defaults: "true"
Expand All @@ -179,6 +202,7 @@ jobs:
- name: Deploy on CVMFS
env:
CVMFS_PROXY_BASE64: ${{ secrets.CVMFS_PROXY_BASE64 }}
# zizmor: ignore[secrets-outside-env] no GitHub environment exists for the CVMFS upload
run: |
eval "$(conda shell.bash hook)" && conda activate cvmfs-env
conda install ca-policy-lcg openssl==3.0.7 gct
Expand All @@ -194,7 +218,7 @@ jobs:
gsissh -p 1975 -t cvmfs-upload01.gridpp.rl.ac.uk /home/diracsgm/admin/sync_packages.sh -v
- name: setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3
with:
limit-access-to-actor: true

Expand Down
21 changes: 18 additions & 3 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,17 @@ on:
- 'README.rst'
- 'release.notes'

# A new push to the same branch/PR cancels the previous run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
Integration:
name: Integration
runs-on: ubuntu-latest
if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
timeout-minutes: 45
Expand All @@ -41,12 +50,14 @@ jobs:
ARGS: DIRAC_USE_JSON_ENCODE=Yes DIRAC_USE_JSON_DECODE=Yes

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Fail-fast for outdated pipelines
run: .github/workflows/fail-fast.sh
- run: |
git fetch --prune --unshallow
- uses: actions/setup-python@v5
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.11'
- name: Installing dependencies
Expand All @@ -58,7 +69,11 @@ jobs:
requests \
typer
- name: Prepare environment
run: ./integration_tests.py prepare-environment ${{ matrix.ARGS }}
env:
MATRIX_ARGS: ${{ matrix.ARGS }}
# Intentionally unquoted: MATRIX_ARGS is a space-separated list of
# KEY=VALUE flags and must be word-split into separate arguments
run: ./integration_tests.py prepare-environment $MATRIX_ARGS
- name: Install server
run: ./integration_tests.py install-server
- name: Install client
Expand Down
Loading
Loading