Files
nixpkgs/.github/workflows/pr.yml
Wolfgang Walther b523f257ac workflows/eval: test all available versions
With this change, we start running Eval on all available Lix and Nix
versions. Because this requires a lot of resources, this complete test
is only run when `ci/pinned.json` is updated.

The resulting outpaths are checked for consistency with the target
branch. A difference will cause the `report` job to fail, thus blocking
the merge, ensuring Eval consistency for Nixpkgs across different
versions.

This implements a kind of "ratchet style" check: Since we originally
confirmed that the versions currently in Nixpkgs at the time of this
commit match Eval behavior of Nix 2.3, we can ensure consistency with
Nix 2.3 down the road, even without testing for it explicitly.

There had been one regression in Eval consistency for Nix between 2.18
and 2.24 - two tests in `tests.devShellTools` produce different results
between Lix 2.91+ (which was forked from Nix 2.18) and Nix 2.24+. I
assume it's unlikely that such a change would be "fixed" by now, thus I
added an exception for these.

As a bonus, we also present the total time in seconds it takes for Eval
to complete for every tested version in a summary table. This allows us
to easily see performance improvements for Eval due to version updates.
At this stage, this time only includes the "outpaths" step of Eval, but
not the generation of attrpaths beforehand.
2025-08-12 10:14:01 +02:00

167 lines
5.5 KiB
YAML

name: PR
on:
pull_request:
paths:
- .github/actions/get-merge-commit/action.yml
- .github/workflows/build.yml
- .github/workflows/check.yml
- .github/workflows/eval.yml
- .github/workflows/lint.yml
- .github/workflows/pr.yml
- .github/workflows/labels.yml
- .github/workflows/reviewers.yml # needs eval results from the same event type
pull_request_target:
concurrency:
group: pr-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
permissions: {}
jobs:
prepare:
runs-on: ubuntu-24.04-arm
outputs:
baseBranch: ${{ steps.branches.outputs.base }}
headBranch: ${{ steps.branches.outputs.head }}
mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }}
targetSha: ${{ steps.get-merge-commit.outputs.targetSha }}
systems: ${{ steps.systems.outputs.systems }}
touched: ${{ steps.files.outputs.touched }}
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
sparse-checkout: |
.github/actions
ci/supportedBranches.js
ci/supportedSystems.json
- name: Check if the PR can be merged and get the test merge commit
uses: ./.github/actions/get-merge-commit
id: get-merge-commit
- name: Load supported systems
id: systems
run: |
echo "systems=$(jq -c <ci/supportedSystems.json)" >> "$GITHUB_OUTPUT"
- name: Determine branch type
id: branches
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const { classify } = require('./ci/supportedBranches.js')
const { base, head } = context.payload.pull_request
const baseClassification = classify(base.ref)
core.setOutput('base', baseClassification)
core.info('base classification:', baseClassification)
const headClassification =
(base.repo.full_name == head.repo.full_name) ?
classify(head.ref) :
// PRs from forks are always considered WIP.
{ type: ['wip'] }
core.setOutput('head', headClassification)
core.info('head classification:', headClassification)
- name: Determine changed files
id: files
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const files = (await github.paginate(github.rest.pulls.listFiles, {
...context.repo,
pull_number: context.payload.pull_request.number,
per_page: 100,
})).map(file => file.filename)
if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned'])
else core.setOutput('touched', [])
check:
name: Check
needs: [prepare]
uses: ./.github/workflows/check.yml
permissions:
# cherry-picks
pull-requests: write
with:
baseBranch: ${{ needs.prepare.outputs.baseBranch }}
headBranch: ${{ needs.prepare.outputs.headBranch }}
lint:
name: Lint
needs: [prepare]
uses: ./.github/workflows/lint.yml
with:
mergedSha: ${{ needs.prepare.outputs.mergedSha }}
targetSha: ${{ needs.prepare.outputs.targetSha }}
eval:
name: Eval
needs: [prepare]
uses: ./.github/workflows/eval.yml
permissions:
# compare
statuses: write
secrets:
OWNER_APP_PRIVATE_KEY: ${{ secrets.OWNER_APP_PRIVATE_KEY }}
with:
mergedSha: ${{ needs.prepare.outputs.mergedSha }}
targetSha: ${{ needs.prepare.outputs.targetSha }}
systems: ${{ needs.prepare.outputs.systems }}
testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }}
labels:
name: Labels
needs: [prepare, eval]
uses: ./.github/workflows/labels.yml
permissions:
issues: write
pull-requests: write
secrets:
NIXPKGS_CI_APP_PRIVATE_KEY: ${{ secrets.NIXPKGS_CI_APP_PRIVATE_KEY }}
with:
headBranch: ${{ needs.prepare.outputs.headBranch }}
reviewers:
name: Reviewers
needs: [prepare, eval]
if: |
needs.prepare.outputs.targetSha &&
!contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development')
uses: ./.github/workflows/reviewers.yml
secrets:
OWNER_APP_PRIVATE_KEY: ${{ secrets.OWNER_APP_PRIVATE_KEY }}
build:
name: Build
needs: [prepare]
uses: ./.github/workflows/build.yml
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
with:
baseBranch: ${{ needs.prepare.outputs.baseBranch }}
mergedSha: ${{ needs.prepare.outputs.mergedSha }}
# This job's only purpose is to serve as a target for the "Required Status Checks" branch ruleset.
# It "needs" all the jobs that should block merging a PR.
# If they pass, it is skipped — which counts as "success" for purposes of the branch ruleset.
# However, if any of them fail, this job will also fail — thus blocking the branch ruleset.
no-pr-failures:
# Modify this list to add or remove jobs from required status checks.
needs:
- check
- lint
- eval
- build
# WARNING:
# Do NOT change the name of this job, otherwise the rule will not catch it anymore.
# This would prevent all PRs from merging.
name: no PR failures
if: ${{ failure() }}
runs-on: ubuntu-24.04-arm
steps:
- run: exit 1