From 587ca61494d5d4dee02d1c90019a0d9fc959186d Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Mon, 27 Oct 2025 15:16:58 +0100 Subject: [PATCH 1/5] workflows/test: prefix artifacts Once we start running similar jobs in the PR and Merge Queue workflows, the Test workflow will run into a problem: Both nested workflows will try to upload the same artifacts, which errors out. By providing a prefix to use for artifact names, this will be a non-issue when we later run more of Eval in the merge queue or when we add the build jobs to the queue as well. (cherry picked from commit 67506595acc2c2103b9edcf464f2b1f8ae3d36c1) --- .github/workflows/build.yml | 5 ++++- .github/workflows/eval.yml | 13 +++++++++---- .github/workflows/merge-group.yml | 4 ++++ .github/workflows/pr.yml | 8 ++++++++ .github/workflows/reviewers.yml | 10 ++++++++-- .github/workflows/test.yml | 3 +++ 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2062ccd7c14a..69bbc48c9099 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,9 @@ name: Build on: workflow_call: inputs: + artifact-prefix: + required: true + type: string baseBranch: required: true type: string @@ -100,5 +103,5 @@ jobs: contains(fromJSON(inputs.baseBranch).type, 'primary') uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: - name: nixos-manual-${{ matrix.name }} + name: ${{ inputs.artifact-prefix }}nixos-manual-${{ matrix.name }} path: nixos-manual diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index d6e6ca7a05ba..4f950b5dffd3 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -3,6 +3,9 @@ name: Eval on: workflow_call: inputs: + artifact-prefix: + required: true + type: string mergedSha: required: true type: string @@ -145,7 +148,7 @@ jobs: if: inputs.targetSha uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: - name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} + name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} path: diff/* compare: @@ -169,7 +172,7 @@ jobs: - name: Download output paths and eval stats for all systems uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: - pattern: diff-* + pattern: ${{ inputs.artifact-prefix }}diff-* path: diff merge-multiple: true @@ -202,7 +205,7 @@ jobs: - name: Upload the comparison results uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: - name: comparison + name: ${{ inputs.artifact-prefix }}comparison path: comparison/* - name: Add eval summary to commit statuses @@ -250,6 +253,7 @@ jobs: - name: Add version comparison table to job summary uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: + ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }} SYSTEMS: ${{ inputs.systems }} VERSIONS: ${{ needs.versions.outputs.versions }} with: @@ -257,6 +261,7 @@ jobs: const { readFileSync } = require('node:fs') const path = require('node:path') + const prefix = process.env.ARTIFACT_PREFIX const systems = JSON.parse(process.env.SYSTEMS) const versions = JSON.parse(process.env.VERSIONS) @@ -272,7 +277,7 @@ jobs: [{ data: version }].concat( systems.map((system) => { try { - const artifact = path.join('versions', `${version}-diff-${system}`) + const artifact = path.join('versions', `${prefix}${version}-diff-${system}`) const time = Math.round( parseFloat( readFileSync( diff --git a/.github/workflows/merge-group.yml b/.github/workflows/merge-group.yml index 24ab1095ea7d..32fe9130b835 100644 --- a/.github/workflows/merge-group.yml +++ b/.github/workflows/merge-group.yml @@ -4,6 +4,9 @@ on: merge_group: workflow_call: inputs: + artifact-prefix: + required: true + type: string mergedSha: required: true type: string @@ -54,6 +57,7 @@ jobs: secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: + artifact-prefix: ${{ inputs.artifact-prefix }} mergedSha: ${{ inputs.mergedSha || github.event.merge_group.head_sha }} systems: ${{ needs.prepare.outputs.systems }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 56017f0b4943..8a305a953933 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -3,6 +3,10 @@ name: PR on: pull_request_target: workflow_call: + inputs: + artifact-prefix: + required: true + type: string secrets: CACHIX_AUTH_TOKEN: required: true @@ -90,6 +94,7 @@ jobs: secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: + artifact-prefix: ${{ inputs.artifact-prefix }} mergedSha: ${{ needs.prepare.outputs.mergedSha }} targetSha: ${{ needs.prepare.outputs.targetSha }} systems: ${{ needs.prepare.outputs.systems }} @@ -116,6 +121,8 @@ jobs: uses: ./.github/workflows/reviewers.yml secrets: OWNER_APP_PRIVATE_KEY: ${{ secrets.OWNER_APP_PRIVATE_KEY }} + with: + artifact-prefix: ${{ inputs.artifact-prefix }} build: name: Build @@ -124,6 +131,7 @@ jobs: secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: + artifact-prefix: ${{ inputs.artifact-prefix }} baseBranch: ${{ needs.prepare.outputs.baseBranch }} mergedSha: ${{ needs.prepare.outputs.mergedSha }} diff --git a/.github/workflows/reviewers.yml b/.github/workflows/reviewers.yml index 6d23e1dde9b7..b5d0ada43b9d 100644 --- a/.github/workflows/reviewers.yml +++ b/.github/workflows/reviewers.yml @@ -7,6 +7,10 @@ on: pull_request_target: types: [ready_for_review] workflow_call: + inputs: + artifact-prefix: + required: true + type: string secrets: OWNER_APP_PRIVATE_KEY: required: true @@ -91,6 +95,8 @@ jobs: - name: Wait for comparison to be done uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 id: eval + env: + ARTIFACT: ${{ inputs.artifact-prefix }}comparison with: script: | const run_id = (await github.rest.actions.listWorkflowRuns({ @@ -111,7 +117,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, run_id, - name: 'comparison' + name: process.env.ARTIFACT, }) if (result.data.total_count > 0) return await new Promise(resolve => setTimeout(resolve, 5000)) @@ -128,7 +134,7 @@ jobs: with: run-id: ${{ steps.eval.outputs.run-id }} github-token: ${{ github.token }} - pattern: comparison + pattern: ${{ inputs.artifact-prefix }}comparison path: comparison merge-multiple: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67b416a6dfb2..4cac4b3f0bb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,7 @@ jobs: secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: + artifact-prefix: mg- mergedSha: ${{ needs.prepare.outputs.mergedSha }} targetSha: ${{ needs.prepare.outputs.targetSha }} @@ -95,3 +96,5 @@ jobs: secrets: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} NIXPKGS_CI_APP_PRIVATE_KEY: ${{ secrets.NIXPKGS_CI_APP_PRIVATE_KEY }} + with: + artifact-prefix: pr- From f88490bd99c1567cc897d56d6e70242f9c888b69 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 29 Oct 2025 21:06:21 +0100 Subject: [PATCH 2/5] workflows/merge-group: compare eval against previous queue item This runs the compare step in the merge queue, to get a combined diff against the previous queue item. This can be used as a base for multiple different features in the future, for example preventing accidental mass rebuilds in the merge queue, or diffing of packages' meta. (cherry picked from commit 6ae5f33c8570507704f24920393e5f6ff3dbb441) --- .github/workflows/eval.yml | 27 ++++++++++++++++++--------- .github/workflows/merge-group.yml | 1 + 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 4f950b5dffd3..3a7509622b78 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -113,7 +113,7 @@ jobs: MATRIX_SYSTEM: ${{ matrix.system }} MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} run: | - nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \ + nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ --argstr nixPath "$MATRIX_VERSION" \ @@ -122,19 +122,29 @@ jobs: # Note: Keep the same further down in sync! - name: Evaluate the ${{ matrix.system }} output paths at the target commit - if: inputs.targetSha env: MATRIX_SYSTEM: ${{ matrix.system }} - # This is very quick, because it pulls the eval results from Cachix. run: | - nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \ + TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ - --argstr nixPath "nixVersions.latest" \ - --out-link target + --argstr nixPath "nixVersions.latest") + + # Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval + # twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier. + for _i in {1..6}; do + # Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix. + if nix-build "$TARGET_DRV" --max-jobs 0; then + break + fi + sleep 5 + done + + # Either fetches from Cachix or runs Eval itself. The fallback is required + # for pull requests into wip-branches without merge queue. + nix-build "$TARGET_DRV" --out-link target - name: Compare outpaths against the target branch - if: inputs.targetSha env: MATRIX_SYSTEM: ${{ matrix.system }} run: | @@ -145,7 +155,6 @@ jobs: --out-link diff - name: Upload outpaths diff and stats - if: inputs.targetSha uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} @@ -154,7 +163,7 @@ jobs: compare: runs-on: ubuntu-24.04-arm needs: [eval] - if: inputs.targetSha && !cancelled() && !failure() + if: ${{ !cancelled() && !failure() }} permissions: statuses: write timeout-minutes: 5 diff --git a/.github/workflows/merge-group.yml b/.github/workflows/merge-group.yml index 32fe9130b835..fe3d8fcc27d0 100644 --- a/.github/workflows/merge-group.yml +++ b/.github/workflows/merge-group.yml @@ -59,6 +59,7 @@ jobs: with: artifact-prefix: ${{ inputs.artifact-prefix }} mergedSha: ${{ inputs.mergedSha || github.event.merge_group.head_sha }} + targetSha: ${{ inputs.targetSha || github.event.merge_group.base_sha }} systems: ${{ needs.prepare.outputs.systems }} # This job's only purpose is to create the target for the "Required Status Checks" branch ruleset. From 4629844e0c11ef2e94bf626071cb2e33e0694499 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 30 Oct 2025 08:42:11 +0100 Subject: [PATCH 3/5] ci/eval/diff: avoid copying paths.json Currently the `diff-` artifacts are 6-7 MB in size - and almost all of that is the `paths.json` file, which is only used to generate the diff itself. This had been stored in the artifact previously for debugging purposes. Ever since we moved to Cachix this is not required anymore, since it's possible to run the same eval locally and thus fetch the `eval.singleSystem` result, including `paths.json`, from Cachix. This will be even more helpful when the next step adds `meta.json` - which is magnitudes bigger than `paths.json`. (cherry picked from commit 350a469f731122997b3a54b876f9b3a59fa0b63d) --- ci/eval/diff.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/eval/diff.nix b/ci/eval/diff.nix index 692e2ec60194..0f757ce3dd12 100644 --- a/ci/eval/diff.nix +++ b/ci/eval/diff.nix @@ -80,7 +80,10 @@ in runCommand "diff" { } '' mkdir -p $out/${evalSystem} - cp -r ${before} $out/before - cp -r ${after} $out/after + cp -r --no-preserve=mode ${before} $out/before + cp -r --no-preserve=mode ${after} $out/after + # JSON files will be processed above explicitly, so avoid copying over + # the source files to keep the artifacts smaller. + find $out/before $out/after -iname '*.json' -delete cp ${diffJson} $out/${evalSystem}/diff.json '' From 200b74373ad74f0a58edfe6a5c29ced13152ccb4 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 30 Oct 2025 08:58:45 +0100 Subject: [PATCH 4/5] ci/eval/utils: refactor to use lib.init (cherry picked from commit 5c01bd9df67fe9590e337d1551ef29eeb8c98835) --- ci/eval/compare/utils.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix index 65a558d8be60..969622ee16f1 100644 --- a/ci/eval/compare/utils.nix +++ b/ci/eval/compare/utils.nix @@ -22,7 +22,7 @@ rec { splittedPath = lib.splitString "." packagePlatformPath; # ["python312Packages" "numpy" "aarch64-linux"] -> ["python312Packages" "numpy"] - packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath; + packagePath = lib.init splittedPath; # "python312Packages.numpy" name = lib.concatStringsSep "." packagePath; From 8311fc26fd3f8dccc5196b76dd8b05580a05dfe7 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 30 Oct 2025 11:22:27 +0100 Subject: [PATCH 5/5] workflows/eval: upload maintainer list of all packages as artifact This creates a simple mapping of all packages to github ids of all their maintainers. This is uploaded as an artifact and is then available for download on the latest commit of each branch with a merge queue. This will allow scheduled jobs to use this information for setting maintainer-related labels, to request reviewers and to implement the merge-bot. The advantage over querying this information directly via Nix in each case: The scheduled job does not need to install Nix and does not need to checkout the target branch. Compared to obtaining the maintainer information just for a single PR during Eval, this will allow retroactively changing maintainers for a package: For example, it allows to request a new maintainer as reviewer for a PR that was created before they became maintainer, but is still open - and similarly for maintainer labels and merge-bot rights. None of these extensions are implemented by this PR, yet. (cherry picked from commit 498574f238960ab7401cb91138036ac5c98955ba) --- .github/workflows/eval.yml | 6 ++++++ ci/eval/default.nix | 10 ++++++++++ ci/eval/diff.nix | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 3a7509622b78..b037b29c9558 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -194,6 +194,12 @@ jobs: --arg diffDir ./diff \ --out-link combined + - name: Upload the maintainer list + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: ${{ inputs.artifact-prefix }}maintainers + path: combined/maintainers.json + - name: Compare against the target branch env: AUTHOR_ID: ${{ github.event.pull_request.user.id }} diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 9cc5ad6857b7..75b2eb9cba5d 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -111,6 +111,7 @@ let --option allow-import-from-derivation false \ --query --available \ --out-path --json \ + --meta \ --show-trace \ --arg chunkSize "$chunkSize" \ --arg myChunk "$myChunk" \ @@ -204,6 +205,7 @@ let fi cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.outputs)' > $out/${evalSystem}/paths.json + cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.meta)' > $out/${evalSystem}/meta.json ''; diff = callPackage ./diff.nix { }; @@ -232,6 +234,14 @@ let }) ' > $out/combined-diff.json + # Combine maintainers from all systems + cat ${diffDir}/*/maintainers.json | jq -s ' + add | group_by(.package) | map({ + key: .[0].package, + value: map(.maintainers) | flatten | unique + }) | from_entries + ' > $out/maintainers.json + mkdir -p $out/before/stats for d in ${diffDir}/before/*; do cp -r "$d"/stats-by-chunk $out/before/stats/$(basename "$d") diff --git a/ci/eval/diff.nix b/ci/eval/diff.nix index 0f757ce3dd12..bb8ba8088c7f 100644 --- a/ci/eval/diff.nix +++ b/ci/eval/diff.nix @@ -76,6 +76,27 @@ let afterAttrs = getAttrs after; diffAttrs = diff beforeAttrs afterAttrs; diffJson = writeText "diff.json" (builtins.toJSON diffAttrs); + + # The maintainer list is not diffed, but just taken as is, to provide a map + # of maintainers on the target branch. A list of GitHub IDs is sufficient for + # all our purposes and reduces size massively. + meta = lib.importJSON "${after}/${evalSystem}/meta.json"; + maintainers = lib.pipe meta [ + (lib.mapAttrsToList ( + k: v: { + # splits off the platform suffix + package = lib.pipe k [ + (lib.splitString ".") + lib.init + (lib.concatStringsSep ".") + ]; + maintainers = map (m: m.githubId) v.maintainers or [ ]; + } + )) + # Some paths don't have a platform suffix, those will appear with an empty package here. + (lib.filter ({ package, maintainers }: package != "" && maintainers != [ ])) + ]; + maintainersJson = writeText "maintainers.json" (builtins.toJSON maintainers); in runCommand "diff" { } '' mkdir -p $out/${evalSystem} @@ -86,4 +107,5 @@ runCommand "diff" { } '' # the source files to keep the artifacts smaller. find $out/before $out/after -iname '*.json' -delete cp ${diffJson} $out/${evalSystem}/diff.json + cp ${maintainersJson} $out/${evalSystem}/maintainers.json ''