From 4a901891c2b644d61e0b7e8f3369f3c850b375cc Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Fri, 6 Nov 2026 10:13:22 -0700 Subject: [PATCH 1/2] Build and test conda packages and wheels Adds a ci/ directory and .github/workflows/intel-ci.yml. The workflow builds conda-recipe/ with conda-build and the wheels with cibuildwheel, for every platform and Python version the recipe and pyproject.toml declare, and tests each package in a clean environment with ci/test.py. conda-recipe/ is now the only recipe, the conda-forge one that was conda-recipe-cf/; the Intel-channel recipe and conda-package.yml, which copied wheels out through WHEELS_OUTPUT_FOLDER, are removed. conda-package-cf.yml reports the same check names, so the required checks still run. conda_build_config.yaml now lists the Python versions, and numpy: '2', as conda-build otherwise defaults to 1.26. The wheels' run-time requirement on mkl comes from [tool.gh-ci] wheel-requires in pyproject.toml, which ci/build.py adds to each wheel: in [project] it would fail the recipe's pip check, as conda-forge's mkl has no pip metadata. The Agent guidance job, ci/guidance.py, fails a pull request that adds a broken link, path, script or identifier claim to an agent guidance file it changes. Claims that already fail, and untouched files, never fail it. dokimi is installed with the hashes in ci/guidance-requirements.txt. The wheels build against mkl-devel >=2026.2,<2027, the range they require at run time, and the Linux wheels are tagged manylinux_2_28 only, like the mkl wheel. ci/build.py keeps only the $ORIGIN entries of each ELF file's RPATH and RUNPATH in a wheel, as meson-python keeps the isolated build env's absolute lib directory, and refuses a file that would be left with none. ci/hooks.py tests each wheel in a venv made from conda-forge's Python, as pip users install it. Every python: entry in conda_build_config.yaml names its ABI, which ci/matrix.py requires, and every push and tag run gets a concurrency group of its own. --- .github/AGENTS.md | 4 +- .github/copilot-instructions.md | 2 +- .github/workflows/conda-package-cf.yml | 4 +- .github/workflows/conda-package.yml | 460 ------------ .github/workflows/intel-ci.yml | 208 ++++++ AGENTS.md | 6 +- README.md | 1 - ci/README.md | 34 + ci/benchmark.py | 84 +++ ci/build.py | 428 +++++++++++ ci/config.yml | 15 + ci/guidance-requirements.txt | 32 + ci/guidance.py | 316 ++++++++ ci/hooks.py | 452 ++++++++++++ ci/matrix.py | 679 ++++++++++++++++++ ci/test.py | 51 ++ conda-recipe-cf/AGENTS.md | 10 - conda-recipe-cf/bld.bat | 2 - conda-recipe-cf/build.sh | 3 - conda-recipe-cf/conda_build_config.yaml | 16 - conda-recipe-cf/meta.yaml | 59 -- conda-recipe/AGENTS.md | 10 +- conda-recipe/bld.bat | 24 +- conda-recipe/build.sh | 28 +- conda-recipe/conda_build_config.yaml | 21 + conda-recipe/meta.yaml | 7 +- meson.build | 9 +- mkl_fft/tests/test_interfaces.py | 18 +- mkl_fft/tests/third_party/scipy/test_basic.py | 5 +- .../third_party/scipy/test_multithreading.py | 7 +- pyproject.toml | 21 +- 31 files changed, 2388 insertions(+), 628 deletions(-) delete mode 100644 .github/workflows/conda-package.yml create mode 100644 .github/workflows/intel-ci.yml create mode 100644 ci/README.md create mode 100644 ci/benchmark.py create mode 100644 ci/build.py create mode 100644 ci/config.yml create mode 100644 ci/guidance-requirements.txt create mode 100644 ci/guidance.py create mode 100644 ci/hooks.py create mode 100644 ci/matrix.py create mode 100644 ci/test.py delete mode 100644 conda-recipe-cf/AGENTS.md delete mode 100644 conda-recipe-cf/bld.bat delete mode 100644 conda-recipe-cf/build.sh delete mode 100644 conda-recipe-cf/conda_build_config.yaml delete mode 100644 conda-recipe-cf/meta.yaml diff --git a/.github/AGENTS.md b/.github/AGENTS.md index f0d319bc..7dde4507 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -3,8 +3,8 @@ CI/CD workflows and repo automation. ## Workflows (source of truth) -- `conda-package.yml` — Intel channel conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline +- `intel-ci.yml` — Intel CI: builds and tests the conda packages of `conda-recipe` and the wheels, for the platforms and Python versions the recipe and `pyproject.toml` declare (`ci/matrix.py`); see `ci/README.md`; its `Agent guidance` job (`ci/guidance.py`) fails a pull request that adds a broken claim to a changed guidance file - `build-with-clang.yml` — clang compatibility checks - `build_pip.yml` — editable pip build pipeline - `build-with-standard-clang.yml` — standard clang compatibility checks @@ -12,5 +12,5 @@ CI/CD workflows and repo automation. - `openssf-scorecard.yml` — security scanning ## Policy -- Treat workflow YAML as canonical for platform/Python matrices. +- Treat workflow YAML as canonical for platform/Python matrices, except `intel-ci.yml`, whose conda matrix is `conda-recipe/conda_build_config.yaml` and whose wheel matrix is the Python classifiers in `pyproject.toml`. - Avoid doc claims about CI coverage unless present in workflow config. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ded9e95f..6d075e54 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -35,7 +35,7 @@ Higher-precedence file overrides lower-precedence context. ## Source-of-truth files - Build/config: `pyproject.toml`, `meson.build` -- Dependencies: `pyproject.toml`, `conda-recipe/meta.yaml`, `conda-recipe-cf/meta.yaml` +- Dependencies: `pyproject.toml`, `conda-recipe/meta.yaml` - CI: `.github/workflows/*.{yml,yaml}` - API: `mkl_fft/__init__.py`, `mkl_fft/interfaces/*.py`, `mkl_fft/_pydfti.pyx` (interface wrappers are integration/patch entry points) - Tests: `mkl_fft/tests/` diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml index 229b5f32..31df20da 100644 --- a/.github/workflows/conda-package-cf.yml +++ b/.github/workflows/conda-package-cf.yml @@ -85,7 +85,7 @@ jobs: "${TEST[@]}" \ "${VERSIONS[@]}" \ "${CHANNELS[@]}" \ - conda-recipe-cf + conda-recipe - name: Upload artifact uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -248,7 +248,7 @@ jobs: - name: Build conda package with NumPy 2.x run: | - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c conda-forge --override-channels conda-recipe-cf + conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c conda-forge --override-channels conda-recipe - name: Store conda paths as envs shell: bash -l {0} diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index e5329b45..00000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,460 +0,0 @@ -name: Conda package - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: mkl_fft - MODULE_NAME: mkl_fft - TEST_ENV_NAME: test_mkl_fft - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['mkl_fft'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - CONDA_BUILD_VERSION: 26.3.0 - -jobs: - build_linux: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Store conda paths as envs - run: | - echo "CONDA_BLD=$CONDA/conda-bld/linux-64/" >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE/" >> "$GITHUB_ENV" - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-build - run: conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Build conda package - run: | - CHANNELS=(-c https://software.repos.intel.com/python/conda -c conda-forge --override-channels) - VERSIONS=(--python "${{ matrix.python_spec || matrix.python }}" --numpy "${{ matrix.numpy }}") - TEST=(--no-test) - - conda build \ - "${TEST[@]}" \ - "${VERSIONS[@]}" \ - "${CHANNELS[@]}" \ - conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [ubuntu-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - env: - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-index - run: | - conda install -n base conda-index -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE"/channel/linux-64 - mv "$PACKAGE_NAME"-*.conda "$GITHUB_WORKSPACE"/channel/linux-64 - conda index "$GITHUB_WORKSPACE"/channel - - - name: Test conda channel - run: | - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE"/channel --override-channels --info --json > "$GITHUB_WORKSPACE"/ver.json - cat "$GITHUB_WORKSPACE"/ver.json - - - name: Collect dependencies - run: | - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - export PACKAGE_VERSION - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - conda create -n ${{ env.TEST_ENV_NAME }} "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "${CHANNELS[@]}" --only-deps --dry-run > lockfile - - - name: Display lockfile - run: cat lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - export PACKAGE_VERSION - conda create -n ${{ env.TEST_ENV_NAME }} "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} pytest "${CHANNELS[@]}" - conda install -n ${{ env.TEST_ENV_NAME }} "scipy>=1.10" "mkl-service" "${CHANNELS[@]}" - # Test installed packages - conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - run: | - source "$CONDA"/etc/profile.d/conda.sh - conda activate ${{ env.TEST_ENV_NAME }} - pytest -v --pyargs "$MODULE_NAME" - - build_windows: - runs-on: windows-latest - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE\\" >> "$GITHUB_ENV" - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Build conda package - run: | - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [windows-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.TEST_ENV_NAME }} - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel\ - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64 - - - name: Index the channel - shell: cmd /C CALL {0} - run: conda index ${{ env.workdir }}\channel - - - name: Dump mkl_fft version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "TEST_DEPENDENCIES=pytest" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %TEST_DEPENDENCIES% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - name: Install additional test dependencies - shell: cmd /C CALL {0} - run: | - conda install -n ${{ env.TEST_ENV_NAME }} "scipy>=1.10" "mkl-service" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - conda info && conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 00000000..dd715060 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,208 @@ +# Builds and tests mkl_fft's conda packages and wheels, see ci/README.md +name: Intel CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +# A push or tag run gets a group of its own, so a queued master push is never dropped. +concurrency: + group: intel-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + + # The check branch protection requires: new broken claims in changed agent guidance fail it. + guidance: + name: Agent guidance + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Scan the changed agent guidance + run: python ci/guidance.py + + # macOS runs on Intel runners, as MKL has no Apple silicon build. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Pull requests keep them 3 days, tags 30. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_fft ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_fft ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" diff --git a/AGENTS.md b/AGENTS.md index 2b9bc5e4..2d831469 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,8 @@ It provides accelerated FFT transforms while aiming to preserve upstream API beh - **Interface adapters:** `mkl_fft/interfaces/` - **Tests:** `mkl_fft/tests/` - **Vendored helpers:** `_vendored/` -- **Packaging:** `conda-recipe/`, `conda-recipe-cf/` +- **Packaging:** `conda-recipe/` for conda, `pyproject.toml` for wheels +- **Intel CI:** `ci/`, `.github/workflows/intel-ci.yml`; see `ci/README.md` - **Benchmarks:** `benchmarks/` ## Build/runtime basics @@ -44,7 +45,6 @@ Use nearest local `AGENTS.md` when present: - `mkl_fft/interfaces/AGENTS.md` — NumPy/SciPy interface adapters - `mkl_fft/src/AGENTS.md` — C template backend rules - `mkl_fft/tests/AGENTS.md` — test scope/conventions -- `conda-recipe/AGENTS.md` — Intel-channel conda packaging -- `conda-recipe-cf/AGENTS.md` — conda-forge recipe context +- `conda-recipe/AGENTS.md` — conda-forge recipe context - `_vendored/AGENTS.md` — vendored tooling boundaries - `benchmarks/AGENTS.md` — ASV performance suite diff --git a/README.md b/README.md index de9810dc..9dbf36e8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![Conda package](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package.yml) [![Editable build using pip and pre-release NumPy](https://github.com/IntelPython/mkl_fft/actions/workflows/build_pip.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/build_pip.yml) [![Conda package with conda-forge channel only](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package-cf.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package-cf.yml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/mkl_fft/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/mkl_fft) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..d22c8928 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,34 @@ +# How Intel CI tests mkl_fft + +`.github/workflows/intel-ci.yml` builds mkl_fft's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux, Windows and Intel macOS unless a platform-only skip in `meta.yaml`, such as `skip: true # [win]`, rules one out. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `mkl_fft Python `, and one builds `mkl_fft Wheels Python `. Then one job per package tests it. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. The ELF files in a wheel keep only their `$ORIGIN` RPATH and RUNPATH entries. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel goes into a venv made from conda-forge's Python, with everything else from PyPI. | +| `guidance.py` | The workflow's `Agent guidance` job, the check branch protection requires. A pull request that changes no agent guidance file passes. Otherwise it scans the merge base and the head with dokimi, installed with the hashes in `guidance-requirements.txt`, and fails on a claim that is new in a changed file and fails at the head: a broken link, path, script or identifier. Claims that already fail do not fail it, but a repeated copy of one does. The hashes cover Linux x86_64 and CPython 3.11 to 3.14, so `GH_CI_RUNNER` must be such a runner or the hashes need extending. | +| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | + +The scripts need only the Python standard library. macOS packages build and test on Intel runners, as MKL has no Apple silicon build. + +**A pull request into `master`.** Actions builds the oldest and the newest regular Python versions on Linux, Windows and macOS, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. + +**A push to `master` or a tag.** Actions builds and tests every declared version. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +python ci/guidance.py --base REV --head REV # the Agent guidance job, for a pull request +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 00000000..6cb356bb --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,84 @@ +""" +Benchmarks the installed package. + +Runs the asv suite in benchmarks/ against the package installed in this +environment, from an empty working directory, without a build of its own, and +records the results under the commit being tested, in $GH_CI_RESULTS. A pull +request samples each benchmark once, enough to show that every one still runs; +a tag measures them properly. A benchmark that fails fails the run; a slower one +does not, as nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 00000000..a2ee8bf5 --- /dev/null +++ b/ci/build.py @@ -0,0 +1,428 @@ +""" +Builds this repository's packages for one Python version. + +For the intel-ci.yml GitHub Actions workflow. + +This file and ci/matrix.py work out the build system from the files present, +and the Python versions from that build system's own declaration: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build. Needs only the Python standard library. +rattler-build is used from PATH, or downloaded at a pinned version for Linux or +Windows and checked against its published sha256; conda-build, or cibuildwheel +for wheels, must already be installed for this interpreter, so run it with the +conda base install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. It also cuts the RPATH and +RUNPATH of each ELF file in a wheel to their $ORIGIN entries, dropping absolute +ones such as the isolated build environment's lib directory, which meson-python +keeps, and refuses a file that would have none left. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import shlex +import shutil +import struct +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} +# The ELF program header and dynamic section values the RUNPATH check reads. +PT_LOAD, PT_DYNAMIC = 1, 2 +DT_NULL, DT_STRTAB, DT_RPATH, DT_RUNPATH = 0, 5, 15, 29 +ORIGIN_PREFIXES = ("$ORIGIN", "${ORIGIN}") + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: + """ + The RPATH and RUNPATH strings of an ELF file, as (file offset, length, + value); none for a file that is not ELF. + """ + if data[:4] != b"\x7fELF": + return [] + if data[4:6] != b"\x02\x01": + raise BuildError(f"{name}: only 64-bit little-endian ELF is read") + (phoff,) = struct.unpack_from(" tuple[bytes, list[str]]: + """ + An ELF file with its RPATH and RUNPATH cut to their $ORIGIN entries, in + place as patchelf does, and the entries it dropped. + """ + edited, dropped = bytearray(data), [] + for start, length, value in elf_search_paths(data, name): + paths = value.split(":") + kept = [path for path in paths if path.startswith(ORIGIN_PREFIXES)] + if kept == paths: + continue + # An empty RUNPATH would mean the working directory + if not kept: + raise BuildError(f"{name}: no $ORIGIN entry in RUNPATH {value!r}") + edited[start : start + length] = ( + ":".join(kept).encode().ljust(length, b"\0") + ) + dropped += [path for path in paths if path not in kept] + return bytes(edited), dropped + + +def with_requirements(metadata: bytes, requirements: list[str]) -> bytes: + """A wheel's METADATA with Requires-Dist lines added.""" + text = metadata.decode("utf-8") + # Some Windows build backends write the metadata with CRLF. + eol = "\r\n" if "\r\n" in text else "\n" + head, blank, body = text.partition(eol * 2) + head = head.rstrip("\r\n") + "".join( + f"{eol}Requires-Dist: {r}" for r in requirements + ) + return (head + (eol * 2 + body if blank else eol)).encode("utf-8") + + +def rewrite_wheel(wheel: Path, requirements: list[str]) -> list[str]: + """ + Adds Requires-Dist lines to a built wheel and cuts its ELF files' RPATH + and RUNPATH to their $ORIGIN entries, updating its RECORD; lists the + entries it dropped. + """ + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + changed, dropped = {}, [] + for index, (info, data) in enumerate(members): + if info.filename == metadata and requirements: + new = with_requirements(data, requirements) + else: + new, paths = origin_only(data, info.filename) + dropped += [f"{path} from {info.filename}" for path in paths] + if new != data: + members[index] = (info, new) + changed[info.filename] = new + if not changed: + return dropped + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{name},{record_hash(changed[name])},{len(changed[name])}" + if (name := line.split(",")[0]) in changed + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + return dropped + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """Moves built packages into output, flat, and lists the ones it moved.""" + output.mkdir(parents=True, exist_ok=True) + built = [] + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + built.append(output / package.name) + shutil.move(str(package), built[-1]) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) / "ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel": + requires = wheel_requires() + for wheel in built: + for dropped in rewrite_wheel(wheel, requires): + print(f"build: dropped RUNPATH entry {dropped}") + if requires: + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 00000000..58b221a1 --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,15 @@ +# Intel CI settings for mkl_fft, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "mkl_fft {platform} Python {python}" + wheel: "mkl_fft {platform} Wheels Python {python}" +test: + requires: [pytest, "scipy >=1.10", mkl-service] + # PyPI has no mkl-service for Python 3.15 yet + wheel_requires: [pytest, "scipy >=1.10", "mkl-service; python_version < '3.15'"] +benchmark: + # PyYAML, which asv needs, has no PyPI wheel for every Python the recipe declares. + requires: [psutil, "scipy >=1.10", mkl-service, pyyaml] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/guidance-requirements.txt b/ci/guidance-requirements.txt new file mode 100644 index 00000000..92020aab --- /dev/null +++ b/ci/guidance-requirements.txt @@ -0,0 +1,32 @@ +# Hash-pinned install for ci/guidance.py: dokimi 0.1.0 and what `dokimi repo scan` imports. +# Wheels only: pure Python, or CPython 3.11-3.14 on Linux x86_64 (manylinux). +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +dokimi==0.1.0 \ + --hash=sha256:4e288728f3e9d666850b4b76847a0b79e2fabbc570b538e4056dfa1d71f63579 +jsonschema==4.26.0 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +regex==2026.9.29 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c +rpds-py==2026.9.1 \ + --hash=sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e \ + --hash=sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff \ + --hash=sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b \ + --hash=sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab \ + --hash=sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd +typing_extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 diff --git a/ci/guidance.py b/ci/guidance.py new file mode 100644 index 00000000..ca417324 --- /dev/null +++ b/ci/guidance.py @@ -0,0 +1,316 @@ +""" +Fails a pull request that adds a broken claim to the agent guidance files. + +For the intel-ci.yml GitHub Actions workflow's "Agent guidance" job. + +Guidance files are AGENTS.md and CLAUDE.md at any depth, +.github/copilot-instructions.md, .github/instructions/, SKILL.md, +.claude/skills/ and skills/. + +Usage: + python ci/guidance.py [--event pull_request|push] [--base REV] [--head REV] + [--work DIR] + +On a pull_request, with no guidance file changed between the merge base of the +pull request and its head, the job passes. Otherwise it installs dokimi with +hashes, scans the merge base and the head with `dokimi repo scan` (no network, +no model), and fails on a claim that fails at the head, in a changed guidance +file, and did not fail in that file at the merge base. A claim is its file, +kind and text, never its line. Claims that already fail do not fail it, and +neither do untouched files. It also fails when the merge base had guidance and +the head has none. A repeated copy of a failing claim counts as new. On a +push it scans the head, reports, and always passes. A summary goes +to $GITHUB_STEP_SUMMARY. The event, --base and --head default to the +workflow's. Needs only the Python standard library, and Python 3.11 or newer +for dokimi. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import venv +from collections import Counter +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parent.parent +REQUIREMENTS = Path(__file__).resolve().parent / "guidance-requirements.txt" +SCHEMA = "repo_readiness.v1" +# dokimi's default globs, plus skills +GLOBS = [ + "**/AGENTS.md", + "**/CLAUDE.md", + ".github/copilot-instructions.md", + ".github/instructions/**/*.md", + "**/SKILL.md", + ".claude/skills/**/*.md", + "skills/**/*.md", +] +MAX_TEXT = 200 + + +class GuidanceError(Exception): + """Raised when the files or the scan cannot be trusted.""" + + +def is_guidance(path: str) -> bool: + """Whether a repository path is a guidance file.""" + name = PurePosixPath(path).name + return ( + name in ("AGENTS.md", "CLAUDE.md", "SKILL.md") + or path == ".github/copilot-instructions.md" + or ( + path.endswith(".md") + and path.startswith( + (".github/instructions/", ".claude/skills/", "skills/") + ) + ) + ) + + +def git(*args: str) -> str: + """Output of git run in this repository.""" + done = subprocess.run( + ["git", "-C", str(ROOT), *args], + check=False, + capture_output=True, + text=True, + ) + if done.returncode: + raise GuidanceError( + f"git {' '.join(args)} failed: {done.stderr.strip()}; " + "the checkout needs fetch-depth: 0" + ) + return done.stdout + + +def has_file(rev: str, name: str) -> bool: + """Whether the tree at rev holds a file.""" + return ( + subprocess.run( + ["git", "-C", str(ROOT), "cat-file", "-e", f"{rev}:{name}"], + check=False, + capture_output=True, + ).returncode + == 0 + ) + + +def touched_files(base: str, head: str) -> list[str]: + """Guidance files that differ between the merge base and the head.""" + names = git("diff", "--name-only", "--no-renames", "-z", base, head) + return sorted(path for path in names.split("\0") if is_guidance(path)) + + +def install_dokimi(work: Path) -> Path: + """Installs the pinned dokimi into a new environment; its executable.""" + env = work / "venv" + venv.create(env, with_pip=True) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + command = [ + str(bindir / "python"), + "-m", + "pip", + "install", + "--quiet", + "--require-hashes", + "--no-deps", + "--only-binary", + ":all:", + "--requirement", + str(REQUIREMENTS), + ] + print(f"+ {' '.join(command)}", flush=True) + subprocess.run(command, check=True) + return bindir / "dokimi" + + +def write_manifest(work: Path, base: str) -> Path: + """The guidance-only manifest for both scans, from the base tree.""" + guidance = {"globs": GLOBS} + if has_file(base, "AGENTS.md"): + guidance["also_scan"] = ["AGENTS.md"] + manifest = { + "schema": "repo_manifest.v1", + "id": "guidance", + "guidance": guidance, + } + path = work / "manifest.yaml" + path.write_text(json.dumps(manifest), encoding="utf-8") + return path + + +def scan( + dokimi: Path, work: Path, manifest: Path, label: str, rev: str +) -> dict: + """The dokimi report of the repository at one revision.""" + out = work / f"{label}.json" + command = [ + str(dokimi), + "repo", + "scan", + f"{ROOT}@{rev}", + "--manifest-file", + str(manifest), + "--cache-dir", + str(work / "cache"), + "--out", + str(out), + ] + print(f"+ {' '.join(command)}", flush=True) + subprocess.run(command, check=True) + report = json.loads(out.read_text(encoding="utf-8")) + if report.get("schema_version") != SCHEMA: + raise GuidanceError( + f"dokimi wrote {report.get('schema_version')!r}, not {SCHEMA}" + ) + return report + + +def failing(report: dict) -> list[dict]: + """The claims that fail in a report.""" + return [c for c in report["claims"] if c["status"] == "fail"] + + +def key(claim: dict) -> tuple[str, str, str]: + """What identifies a claim across revisions.""" + return claim["file"], claim["kind"], claim["claim"] + + +def new_failures(base: dict, head: dict, touched: list[str]) -> list[dict]: + """Failing head claims in touched files that did not fail at the base.""" + old = Counter(key(claim) for claim in failing(base)) + found = [] + for claim in failing(head): + if claim["file"] not in touched: + continue + if old[key(claim)] > 0: + old[key(claim)] -= 1 + else: + found.append(claim) + return found + + +def code(text: object) -> str: + """Untrusted text as one Markdown code span: no links, no table breaks.""" + text = re.sub(r"\s+", " ", str(text)).strip() + if len(text) > MAX_TEXT: + text = text[: MAX_TEXT - 3] + "..." + text = text.replace("`", "'").replace("|", "\\|") + return f"`{text}`" + + +def plain(text: object) -> str: + """Untrusted text for the log, which cannot start a workflow command.""" + text = re.sub(r"[^\x20-\x7e\n]", "?", str(text)) + return text.replace("::", ": :") + + +def claim_counts(report: dict) -> str: + """One line of a report's counts.""" + counts = report["counts"] + return ( + f"Claims: {counts['total']} " + f"(ok {counts['ok']}, unresolved {counts['unresolved']}, " + f"fail {counts['fail']})." + ) + + +def summarize( + touched: list[str], head: dict, found: list[dict], problem: str +) -> str: + """The Markdown job summary of a scanned pull request.""" + lines = [ + "## Agent guidance", + "", + f"Guidance files changed: {len(touched)}.", + claim_counts(head), + f"New failing claims: {len(found)}.", + ] + if problem: + lines.extend(["", problem]) + if found: + lines.extend( + ["", "| Location | Kind | Claim | Why |", "|---|---|---|---|"] + ) + for claim in found: + location = f"{claim['file']}:{claim['line']}" + lines.append( + f"| {code(location)} | {code(claim['kind'])} " + f"| {code(claim['claim'])} | {code(claim['why'])} |" + ) + return "\n".join([*lines, ""]) + + +def pull_request_range(args: argparse.Namespace) -> tuple[str, str]: + """The pull request's base and head revisions.""" + base, head = args.base, args.head + if not (base and head): + with open(os.environ["GITHUB_EVENT_PATH"], encoding="utf-8") as file: + event = json.load(file)["pull_request"] + base = base or event["base"]["sha"] + head = head or event["head"]["sha"] + return git("merge-base", base, head).strip(), head + + +def check(args: argparse.Namespace, work: Path) -> tuple[str, int]: + """The summary and exit status for one event.""" + if args.event == "push": + head = args.head or os.environ["GITHUB_SHA"] + try: + dokimi = install_dokimi(work) + manifest = write_manifest(work, head) + report = scan(dokimi, work, manifest, "head", head) + except (GuidanceError, subprocess.CalledProcessError) as error: + text = f"Report only, the scan failed: {plain(error)}" + return f"## Agent guidance\n\n{text}\n", 0 + text = f"## Agent guidance\n\nReport only.\n{claim_counts(report)}\n" + return text, 0 + base, head = pull_request_range(args) + touched = touched_files(base, head) + if not touched: + return "## Agent guidance\n\nNo guidance files changed.\n", 0 + dokimi = install_dokimi(work) + manifest = write_manifest(work, base) + old = scan(dokimi, work, manifest, "base", base) + new = scan(dokimi, work, manifest, "head", head) + found = new_failures(old, new, touched) + problem = "" + if new["incomplete"] and not old["incomplete"]: + problem = "The head has no guidance left to scan, the base had some." + return summarize(touched, new, found, problem), int(bool(found or problem)) + + +def main(argv: list[str] | None = None) -> int: + """Runs the gate.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--event", + default=os.environ.get("GITHUB_EVENT_NAME", "pull_request"), + choices=("pull_request", "push"), + ) + parser.add_argument("--base", help="the pull request's base revision") + parser.add_argument("--head", help="the pull request's head revision") + parser.add_argument( + "--work", type=Path, help="scratch directory; default is a new one" + ) + args = parser.parse_args(argv) + with tempfile.TemporaryDirectory() as scratch: + summary, status = check(args, args.work or Path(scratch)) + print(plain(summary)) + target = os.environ.get("GITHUB_STEP_SUMMARY") + if target: + with open(target, "a", encoding="utf-8") as file: + file.write(summary) + return status + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (GuidanceError, subprocess.CalledProcessError) as error: + print(f"guidance: {plain(error)}", file=sys.stderr) + sys.exit(1) diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 00000000..c6897e2f --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,452 @@ +""" +Runs ci/test.py against one built artifact. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel goes into +a venv made from conda-forge's Python, with the test requirements, by pip, as +the wheel's users install it. The test gets the variables ci/test.py reads, and +nothing else from the runner. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json itself, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The conda spec for the test environment's interpreter.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path, kind: str = "conda") -> Path: + """The interpreter of a conda environment, or of a wheel's venv.""" + if not IS_WINDOWS: + return env_dir / "bin" / "python" + if kind == "wheel": + return env_dir / "Scripts" / "python.exe" + return env_dir / "python.exe" + + +def env_path(env_dir: Path, kind: str) -> list[Path]: + """ + The directories activating the conda environment, or a wheel's venv, would + put first on PATH. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + if kind == "wheel": + return [env_dir / "Scripts"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables ci/test.py reads, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir, args.kind)), env.get("PATH", "")] + ) + prefix = "VIRTUAL_ENV" if args.kind == "wheel" else "CONDA_PREFIX" + env.update( + { + prefix: str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [ + str(env_python(env_dir, args.kind)), + "-m", + "pip", + "install", + "--no-input", + ] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + base = args.work / "python" + conda_create(base, python) + run([str(env_python(base)), "-m", "venv", str(env_dir)]) + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir, args.kind)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 00000000..5b197bd7 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,679 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix follows it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform in PLATFORMS, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; `osx and arm64` rules out none, as +macOS here is Intel. A skip that also depends on something else, such as +`win and py < 311`, or a skip: line that cannot be read, is refused, as the +matrix would still list the builds it skips. A pyproject.toml without Operating +System classifiers, or with OS Independent, builds on all three. Of the +declared platforms, the matrix holds those GitHub Actions has runners for, +BUILT_PLATFORMS. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions are listed. --github-output also writes NAME= to +$GITHUB_OUTPUT, for later jobs. Standard library only; reading pyproject.toml +needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms CI builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:") +# conda-build's own selector pattern, which lets a comment follow the selector +CONDA_SELECTOR_RE = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant with its ABI: 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t; from 3.13 it must be cp3N or cp3Nt. +CONDA_SPEC_RE = re.compile(r"3\.(\d+)\.\*\s+\*_(?:cpython|cp3(\d+)(t?))") +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one CI can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if ( + not match + or (match[2] and match[2] != match[1]) + or (not match[2] and int(match[1]) >= FREE_THREADED_MINOR) + ): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.12.* *_cpython", "3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """ + Every platform in PLATFORMS, but those a skip rules out; a skip that + depends on more than the platform is refused. + """ + for skip in skips: + if any( + skips_platform(skip, platform) is None for platform in PLATFORMS + ): + raise DeclarationError( + f"{where}: the skip {skip!r} depends on more than the " + "platform, so the matrix would still list the builds it skips" + ) + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines; a skip: line it cannot read is refused. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [] + for line in recipe.read_text(encoding="utf-8").splitlines(): + if not CONDA_SKIP_RE.match(line): + continue + selected = CONDA_SELECTOR_RE.match(line) + key, selector = ( + (selected[1], selected[3]) + if selected + else (line.split(" #")[0], "True") + ) + value = key.split(":", 1)[1].strip() + if value in ("false", "False", "no"): + continue + if value not in ("true", "True", "yes"): + raise DeclarationError( + f"{where}: cannot read {line.strip()!r}; write " + "skip: true # [selector]" + ) + skips.append(selector) + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """All of them, or for a pull request the oldest and newest regular ones.""" + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 00000000..4ecdbe0f --- /dev/null +++ b/ci/test.py @@ -0,0 +1,51 @@ +""" +Tests mkl_fft's installed package. + +Runs with one built conda package or wheel installed in a clean environment, as +ci/hooks.py sets up, from an empty working directory, so the tests import the +installed mkl_fft rather than the source tree. The test requirements come from +ci/config.yml. Writes a JUnit report to $GH_CI_RESULTS. +""" + +import os +import pathlib +import sys +import sysconfig + +import pytest + +PACKAGE = "mkl_fft" + + +def running_python() -> str: + """This interpreter's version as artifact names spell it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def main() -> int: + """Runs the installed package's test suite.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, expected {expected}") + return 1 + # The suite name says which platform and Python ran. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + return pytest.main( + [ + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/AGENTS.md b/conda-recipe-cf/AGENTS.md deleted file mode 100644 index 9d8655e2..00000000 --- a/conda-recipe-cf/AGENTS.md +++ /dev/null @@ -1,10 +0,0 @@ -# AGENTS.md — conda-recipe-cf/ - -Conda-forge recipe context for `mkl_fft`. - -## Scope -- conda-forge specific recipe files under this directory. - -## Guardrails -- Keep conda-forge recipe semantics separate from Intel-channel recipe. -- Align changes with `conda-package-cf.yml` workflow behavior. diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index a08c27bb..00000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -%PYTHON% -m pip install --no-build-isolation --no-deps . -if errorlevel 1 exit 1 diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100644 index 72e5e4e6..00000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -ex - -$PYTHON -m pip install --no-build-isolation --no-deps . diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index 822fd77b..00000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '14' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 47382240..00000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,59 +0,0 @@ -package: - name: mkl_fft - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: ../ - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports: - - blas - - numpy - -requirements: - build: - - {{ compiler('c') }} - - {{ stdlib('c') }} - host: - - meson-python >=0.13.0 - - cmake - - ninja - - python - - pip - - mkl-devel - - cython >=3.1.0 - - numpy - - wheel >=0.41.3 - run: - - python - - {{ pin_compatible('mkl') }} - - numpy - -test: - commands: - - pip check - - pytest -v --pyargs mkl_fft - requires: - - pip - - pytest - - scipy >=1.10 - - mkl-service - imports: - - mkl_fft - - mkl_fft.interfaces - -about: - home: http://github.com/IntelPython/mkl_fft - license: BSD-3-Clause - license_file: LICENSE.txt - summary: NumPy-based implementation of Fast Fourier Transform using Intel® oneAPI Math Kernel Library (oneMKL) - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

diff --git a/conda-recipe/AGENTS.md b/conda-recipe/AGENTS.md index 7c7cef4b..c52bdf79 100644 --- a/conda-recipe/AGENTS.md +++ b/conda-recipe/AGENTS.md @@ -1,11 +1,11 @@ # AGENTS.md — conda-recipe/ -Intel-channel conda packaging context. +The conda recipe for `mkl_fft`, built against conda-forge only. ## Scope -- `meta.yaml` — package metadata and dependency pins -- `build.sh` / `bld.bat` — platform build scripts +- conda-forge specific recipe files under this directory. ## Guardrails -- Treat recipe files as canonical for packaging intent. -- Keep recipe updates synchronized with CI workflow behavior. +- Align changes with `conda-package-cf.yml` workflow behavior. +- `conda_build_config.yaml`'s `python:` list is the Intel CI conda matrix, read by `ci/matrix.py`: one version per line, no selectors. +- Wheels don't come from this recipe: `intel-ci.yml` builds them from `pyproject.toml` with `ci/build.py`. diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 34f84f2d..a08c27bb 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -1,22 +1,2 @@ -:: -wnx flags mean: --wheel --no-isolation --skip-dependency-check -:: -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -%PYTHON% -m build -w -n -x -Ccompile-args=-v -if %ERRORLEVEL% neq 0 exit 1 - -:: wheel file was renamed -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -:: Copy wheel package -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\mkl_fft*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 -) +%PYTHON% -m pip install --no-build-isolation --no-deps . +if errorlevel 1 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 4f94516d..72e5e4e6 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,29 +1,3 @@ #!/bin/bash -ex -if [ -d "build" ]; then - rm -rf build -fi - -read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -# -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -${PYTHON} -m build -w -n -x -Ccompile-args=-v - -${PYTHON} -m wheel tags --remove \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/mkl_fft*.whl - -${PYTHON} -m pip install dist/mkl_fft*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv - -# Copy wheel package -if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then - cp dist/mkl_fft*.whl "${WHEELS_OUTPUT_FOLDER[@]}" -fi +$PYTHON -m pip install --no-build-isolation --no-deps . diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 822fd77b..9c7b72dd 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,3 +1,14 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.10.* *_cpython" + - "3.11.* *_cpython" + - "3.12.* *_cpython" + - "3.13.* *_cp313" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" +numpy: + - '2' c_compiler: # [linux] - gcc # [linux] cxx_compiler: # [linux] @@ -14,3 +25,13 @@ cxx_compiler: # [win] - vs2022 # [win] c_compiler: # [win] - vs2022 # [win] +c_compiler: # [osx] + - clang # [osx] +c_compiler_version: # [osx] + - '21' # [osx] +c_stdlib: # [osx] + - macosx_deployment_target # [osx] +c_stdlib_version: # [osx] + - '11.0' # [osx] +MACOSX_DEPLOYMENT_TARGET: # [osx] + - '11.0' # [osx] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 50a45378..47382240 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -7,8 +7,6 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER ignore_run_exports: - blas - numpy @@ -19,7 +17,6 @@ requirements: - {{ stdlib('c') }} host: - meson-python >=0.13.0 - - python-build >=1.2.2 - cmake - ninja - python @@ -31,12 +28,14 @@ requirements: run: - python - {{ pin_compatible('mkl') }} - - numpy >=1.26.4 + - numpy test: commands: + - pip check - pytest -v --pyargs mkl_fft requires: + - pip - pytest - scipy >=1.10 - mkl-service diff --git a/meson.build b/meson.build index b7b5b932..df712b6f 100644 --- a/meson.build +++ b/meson.build @@ -42,12 +42,19 @@ if host_machine.system() != 'windows' ] endif +# mkl-devel's CMake config is under \Library on Windows, which CMake only +# finds through a Library\bin PATH entry: a conda env has one, a venv doesn't +mkl_cmake_prefix = [] +if host_machine.system() == 'windows' + mkl_cmake_prefix += '-DCMAKE_PREFIX_PATH=' + (py.get_path('data') / 'Library') +endif + mkl_dep = dependency('MKL', method: 'cmake', modules: ['MKL::MKL'], cmake_args: [ '-DMKL_ARCH=intel64', '-DMKL_LINK=sdl', - ], + ] + mkl_cmake_prefix, required: true ) diff --git a/mkl_fft/tests/test_interfaces.py b/mkl_fft/tests/test_interfaces.py index 226d4625..14b8c4a0 100644 --- a/mkl_fft/tests/test_interfaces.py +++ b/mkl_fft/tests/test_interfaces.py @@ -43,12 +43,18 @@ ids.append("numpy") +def require_scipy_fft(): + """Skips the test unless mkl_fft has its SciPy interface.""" + if scipy_fft is None: + pytest.skip("requires scipy and mkl-service") + + @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize( "dtype", [np.float32, np.float64, np.complex64, np.complex128] ) def test_scipy_fft(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(511, dtype=dtype) w = mfi.scipy_fft.fft(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.ifft(w, norm=norm, workers=None, plan=None) @@ -71,7 +77,7 @@ def test_numpy_fft(norm, dtype): @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_scipy_rfft(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(511, dtype=dtype) w = mfi.scipy_fft.rfft(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.irfft( @@ -102,7 +108,7 @@ def test_numpy_rfft(norm, dtype): "dtype", [np.float32, np.float64, np.complex64, np.complex128] ) def test_scipy_fftn(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones((37, 83), dtype=dtype) w = mfi.scipy_fft.fftn(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.ifftn(w, norm=norm, workers=None, plan=None) @@ -125,7 +131,7 @@ def test_numpy_fftn(norm, dtype): @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_scipy_rfftn(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones((37, 83), dtype=dtype) w = mfi.scipy_fft.rfftn(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.irfftn(w, s=x.shape, norm=norm, workers=None, plan=None) @@ -160,13 +166,13 @@ def _get_blacklisted_dtypes(): @pytest.mark.parametrize("dtype", _get_blacklisted_dtypes()) def test_scipy_no_support_for(dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(16, dtype=dtype) assert_raises(NotImplementedError, mfi.scipy_fft.ifft, x) def test_scipy_fft_arg_validate(): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() with pytest.raises(ValueError): mfi.scipy_fft.fft([1, 2, 3, 4], norm=b"invalid") diff --git a/mkl_fft/tests/third_party/scipy/test_basic.py b/mkl_fft/tests/third_party/scipy/test_basic.py index 25cd3d61..a8e5025f 100644 --- a/mkl_fft/tests/third_party/scipy/test_basic.py +++ b/mkl_fft/tests/third_party/scipy/test_basic.py @@ -32,7 +32,10 @@ else: from numpy import permute_dims -import mkl_fft.interfaces.scipy_fft as fft +try: + import mkl_fft.interfaces.scipy_fft as fft +except ImportError: + pytest.skip("This test file needs mkl-service", allow_module_level=True) skip_xp_backends = pytest.mark.skip_xp_backends diff --git a/mkl_fft/tests/third_party/scipy/test_multithreading.py b/mkl_fft/tests/third_party/scipy/test_multithreading.py index f948e221..414c3902 100644 --- a/mkl_fft/tests/third_party/scipy/test_multithreading.py +++ b/mkl_fft/tests/third_party/scipy/test_multithreading.py @@ -3,15 +3,18 @@ import multiprocessing -import mkl import numpy as np import pytest from numpy.testing import assert_allclose try: + import mkl + import mkl_fft.interfaces.scipy_fft as fft except ImportError: - pytest.skip("This test file needs scipy", allow_module_level=True) + pytest.skip( + "This test file needs scipy and mkl-service", allow_module_level=True + ) @pytest.fixture(scope="module") diff --git a/pyproject.toml b/pyproject.toml index 635c7e46..d0d141c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "ninja", "Cython>=3.1.0", "numpy", - "mkl-devel", + "mkl-devel>=2026.2,<2027", "cmake" ] @@ -80,10 +80,29 @@ extend-exclude = "(^|/)_vendored/" force-exclude = "(^|/)_vendored/" line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# MKL comes from the mkl wheel at run time, see [tool.gh-ci] +# Tagged as the mkl wheel is, as auditwheel doesn't check the excluded library +repair-wheel-command = "auditwheel repair --plat manylinux_2_28_x86_64 --only-plat --exclude libmkl_rt.so.3 -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# MSVC, as python.org builds CPython with, not the runner's MinGW +config-settings = { setup-args = "--vsenv" } +# MKL comes from the mkl wheel at run time, see [tool.gh-ci] +repair-wheel-command = "delvewheel repair --no-dll mkl_rt.3.dll -w {dest_dir} {wheel}" + [tool.cython-lint] ignore = ['E722'] # do not use bare 'except' max-line-length = 80 +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe; +# the same MKL as [build-system], whose library names the repair commands use +wheel-requires = ["mkl >=2026.2,<2027"] + [tool.isort] ensure_newline_before_comments = true force_grid_wrap = 0 From 9978c076d0a56fdfbfcd8190708238cc7da03fed Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Fri, 6 Nov 2026 11:30:09 -0700 Subject: [PATCH 2/2] Read x86 and osx64 as conda-build does, and check $ORIGIN exactly conda-build reads x86 as any x86 CPU and has no osx64 selector, while rattler-build has x86 for 32-bit only and osx64 for Intel macOS, so ci/matrix.py now gives conda-build its own values for both and refuses a conda-build skip that uses osx64. ci/build.py keeps a search path entry only when it is $ORIGIN or a directory under it, names RPATH and RUNPATH entries apart, and rewrites the wheels in scratch, so a refused wheel never reaches --output. --- ci/README.md | 2 +- ci/build.py | 43 +++++++++++++++++++++++++++---------------- ci/matrix.py | 42 ++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/ci/README.md b/ci/README.md index d22c8928..e1a48731 100644 --- a/ci/README.md +++ b/ci/README.md @@ -15,7 +15,7 @@ | `test.py` | Runs the installed package's test suite with pytest. | | `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | -The scripts need only the Python standard library. macOS packages build and test on Intel runners, as MKL has no Apple silicon build. +The scripts need nothing beyond the Python standard library and the tools each one's docstring names. macOS packages build and test on Intel runners, as MKL has no Apple silicon build. **A pull request into `master`.** Actions builds the oldest and the newest regular Python versions on Linux, Windows and macOS, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. diff --git a/ci/build.py b/ci/build.py index a2ee8bf5..652f44c6 100644 --- a/ci/build.py +++ b/ci/build.py @@ -63,7 +63,7 @@ # The ELF program header and dynamic section values the RUNPATH check reads. PT_LOAD, PT_DYNAMIC = 1, 2 DT_NULL, DT_STRTAB, DT_RPATH, DT_RUNPATH = 0, 5, 15, 29 -ORIGIN_PREFIXES = ("$ORIGIN", "${ORIGIN}") +ORIGIN = ("$ORIGIN", "${ORIGIN}") # The rattler-build release fetched when none is on PATH, per system, with the # file name it goes under and the sha256 its release publishes. @@ -152,8 +152,8 @@ def record_hash(data: bytes) -> str: def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: """ - The RPATH and RUNPATH strings of an ELF file, as (file offset, length, - value); none for a file that is not ELF. + The RPATH and RUNPATH strings of an ELF file, as (RPATH or RUNPATH, file + offset, length, value); none for a file that is not ELF. """ if data[:4] != b"\x7fELF": return [] @@ -187,28 +187,38 @@ def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: if tag in (DT_RPATH, DT_RUNPATH): start = base + value end = data.index(b"\0", start) - found.append((start, end - start, data[start:end].decode())) + kind = "RPATH" if tag == DT_RPATH else "RUNPATH" + found.append((kind, start, end - start, data[start:end].decode())) return found +def origin_relative(path: str) -> bool: + """Whether a search path entry is $ORIGIN or a directory under it.""" + return any( + path == token or path.startswith(token + "/") for token in ORIGIN + ) + + def origin_only(data: bytes, name: str) -> tuple[bytes, list[str]]: """ An ELF file with its RPATH and RUNPATH cut to their $ORIGIN entries, in place as patchelf does, and the entries it dropped. """ edited, dropped = bytearray(data), [] - for start, length, value in elf_search_paths(data, name): + for kind, start, length, value in elf_search_paths(data, name): paths = value.split(":") - kept = [path for path in paths if path.startswith(ORIGIN_PREFIXES)] + kept = [path for path in paths if origin_relative(path)] if kept == paths: continue - # An empty RUNPATH would mean the working directory + # An empty search path would mean the working directory if not kept: - raise BuildError(f"{name}: no $ORIGIN entry in RUNPATH {value!r}") + raise BuildError(f"{name}: no $ORIGIN entry in {kind} {value!r}") edited[start : start + length] = ( ":".join(kept).encode().ljust(length, b"\0") ) - dropped += [path for path in paths if path not in kept] + dropped += [ + f"{kind} entry {path}" for path in paths if path not in kept + ] return bytes(edited), dropped @@ -393,6 +403,7 @@ def main(argv: list[str] | None = None) -> int: ) if system == "rattler-build": ensure_rattler_build(args.tools) + requires = wheel_requires() if system == "wheel" else [] output = args.output.resolve() with tempfile.TemporaryDirectory() as scratch: for command in commands( @@ -404,14 +415,14 @@ def main(argv: list[str] | None = None) -> int: ): print(f"+ {shlex.join(command)}", flush=True) subprocess.run(command, check=True) + # Rewritten in scratch, so a refused wheel never reaches the output + if system == "wheel": + for wheel in sorted(Path(scratch).rglob("*.whl")): + for dropped in rewrite_wheel(wheel, requires): + print(f"build: dropped {dropped}") built = collect(Path(scratch), output) - if system == "wheel": - requires = wheel_requires() - for wheel in built: - for dropped in rewrite_wheel(wheel, requires): - print(f"build: dropped RUNPATH entry {dropped}") - if requires: - print("build: added Requires-Dist " + ", ".join(requires)) + if requires: + print("build: added Requires-Dist " + ", ".join(requires)) print("build: " + ", ".join(path.name for path in built)) return 0 diff --git a/ci/matrix.py b/ci/matrix.py index 5b197bd7..b4fc8597 100644 --- a/ci/matrix.py +++ b/ci/matrix.py @@ -77,8 +77,8 @@ # The platforms the workflow can build on: GitHub-hosted runners have all # three. A declared platform missing here is reported and left out. BUILT_PLATFORMS = ("linux", "windows", "macos") -# What a platform-only selector means on each platform, for both conda-build and -# rattler-build. +# What a platform-only selector means on each platform, as rattler-build defines +# it; CONDA_BUILD_SELECTORS holds where conda-build reads it differently. SELECTORS = { "linux": { "linux": True, @@ -130,6 +130,9 @@ "s390x": False, }, } +# conda-build reads x86 as any x86 CPU and has no osx64; None refuses a skip +# that uses it. +CONDA_BUILD_SELECTORS = {"x86": True, "osx64": None} CONDA_SKIP_RE = re.compile(r"\s*skip\s*:") # conda-build's own selector pattern, which lets a comment follow the selector CONDA_SELECTOR_RE = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") @@ -333,12 +336,15 @@ def from_pyproject(path: Path) -> list[Python]: return [Python(version, version) for version in versions] -def skips_platform(expression: str, platform: str) -> bool | None: +def skips_platform(expression: str, platform: str, system: str) -> bool | None: """ - Whether a skip expression rules out a whole platform: True or False when it - depends on the platform alone, None when it also depends on anything else, - such as the Python version. + Whether a skip expression rules out a whole platform, as the build system + reads it: True or False when it depends on the platform alone, None when it + also depends on anything else, such as the Python version. """ + names = SELECTORS[platform] | ( + CONDA_BUILD_SELECTORS if system == "conda-build" else {} + ) try: tree = ast.parse( expression.strip().removeprefix("${{").removesuffix("}}").strip(), @@ -353,8 +359,8 @@ def value(node: ast.AST) -> bool: return all(values) if isinstance(node.op, ast.And) else any(values) if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): return not value(node.operand) - if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: - return SELECTORS[platform][node.id] + if isinstance(node, ast.Name) and names.get(node.id) is not None: + return names[node.id] if isinstance(node, ast.Constant) and isinstance(node.value, bool): return node.value raise ValueError(ast.dump(node)) @@ -365,23 +371,25 @@ def value(node: ast.AST) -> bool: return None -def platforms_without(skips: list[str], where: str) -> list[str]: +def platforms_without(skips: list[str], where: str, system: str) -> list[str]: """ Every platform in PLATFORMS, but those a skip rules out; a skip that - depends on more than the platform is refused. + depends on more than the platform, or on a name the build system doesn't + define, is refused. """ for skip in skips: if any( - skips_platform(skip, platform) is None for platform in PLATFORMS + skips_platform(skip, platform, system) is None + for platform in PLATFORMS ): raise DeclarationError( - f"{where}: the skip {skip!r} depends on more than the " - "platform, so the matrix would still list the builds it skips" + f"{where}: cannot tell which platforms the skip {skip!r} rules " + "out, so the matrix would still list the builds it skips" ) kept = [ platform for platform in PLATFORMS - if not any(skips_platform(skip, platform) for skip in skips) + if not any(skips_platform(skip, platform, system) for skip in skips) ] if not kept: raise DeclarationError( @@ -454,7 +462,9 @@ def from_rattler_recipe(recipe: Path) -> list[str]: else str(recipe) ) return platforms_without( - yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), + where, + "rattler-build", ) @@ -487,7 +497,7 @@ def from_conda_recipe(recipe: Path) -> list[str]: "skip: true # [selector]" ) skips.append(selector) - return platforms_without(skips, where) + return platforms_without(skips, where, "conda-build") def from_os_classifiers(path: Path) -> list[str]: