[Backport release-24.11] workflows: cleanup & refactor checkouts (#410825)

.github/actions/get-merge-commit/action.yml

@@ -1,23 +1,31 @@
 name: Get merge commit
 
-description: 'Checks whether the Pull Request is mergeable and returns two commit hashes: The result of a temporary merge of the head branch into the target branch ("merged"), and the parent of that commit on the target branch ("target"). Handles push events and merge conflicts gracefully.'
+description: 'Checks whether the Pull Request is mergeable and checks out the repo at up to two commits: The result of a temporary merge of the head branch into the target branch ("merged"), and the parent of that commit on the target branch ("target"). Handles push events and merge conflicts gracefully.'
+
+inputs:
+  merged-as-untrusted:
+    description: "Whether to checkout the merge commit in the ./untrusted folder."
+    type: boolean
+  target-as-trusted:
+    description: "Whether to checkout the target commit in the ./trusted folder."
+    type: boolean
 
 outputs:
   mergedSha:
     description: "The merge commit SHA"
-    value: ${{ fromJSON(steps.merged.outputs.result).mergedSha }}
+    value: ${{ steps.commits.outputs.mergedSha }}
   targetSha:
     description: "The target commit SHA"
-    value: ${{ fromJSON(steps.merged.outputs.result).targetSha }}
+    value: ${{ steps.commits.outputs.targetSha }}
 
 runs:
   using: composite
   steps:
-    - id: merged
+    - id: commits
       uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
       with:
         script: |
-          if (context.eventName == 'push') return { mergedSha: context.sha }
+          if (context.eventName == 'push') return core.setOutput('mergedSha', context.sha)
 
           for (const retryInterval of [5, 10, 20, 40, 80]) {
             console.log("Checking whether the pull request can be merged...")
@@ -35,32 +43,46 @@ runs:
               continue
             }
 
+            let mergedSha, targetSha
+
             if (prInfo.mergeable) {
               console.log("The PR can be merged.")
 
-              const mergedSha = prInfo.merge_commit_sha
-              const targetSha = (await github.rest.repos.getCommit({
+              mergedSha = prInfo.merge_commit_sha
+              targetSha = (await github.rest.repos.getCommit({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 ref: prInfo.merge_commit_sha
               })).data.parents[0].sha
-
-              console.log(`Checking the commits:\nmerged:${mergedSha}\ntarget:${targetSha}`)
-
-              return { mergedSha, targetSha }
             } else {
               console.log("The PR has a merge conflict.")
 
-              const mergedSha = prInfo.head.sha
-              const targetSha = (await github.rest.repos.compareCommitsWithBasehead({
+              mergedSha = prInfo.head.sha
+              targetSha = (await github.rest.repos.compareCommitsWithBasehead({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 basehead: `${prInfo.base.sha}...${prInfo.head.sha}`
               })).data.merge_base_commit.sha
-
-              console.log(`Checking the commits:\nmerged:${mergedSha}\ntarget:${targetSha}`)
-
-              return { mergedSha, targetSha }
             }
+
+            console.log(`Checking the commits:\nmerged:${mergedSha}\ntarget:${targetSha}`)
+            core.setOutput('mergedSha', mergedSha)
+            core.setOutput('targetSha', targetSha)
+            return
           }
           throw new Error("Not retrying anymore. It's likely that GitHub is having internal issues: check https://www.githubstatus.com.")
+
+      # Would be great to do the checkouts in git worktrees of the existing spare checkout instead,
+      # but Nix is broken with them:
+      # https://github.com/NixOS/nix/issues/6073
+    - if: inputs.merged-as-untrusted && steps.commits.outputs.mergedSha
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        ref: ${{ steps.commits.outputs.mergedSha }}
+        path: untrusted
+
+    - if: inputs.target-as-trusted && steps.commits.outputs.targetSha
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        ref: ${{ steps.commits.outputs.targetSha }}
+        path: trusted

.github/workflows/check-cherry-picks.yml

@@ -21,10 +21,11 @@ jobs:
         with:
           fetch-depth: 0
           filter: blob:none
+          path: trusted
 
       - name: Check cherry-picks
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          ./maintainers/scripts/check-cherry-picks.sh "$BASE_SHA" "$HEAD_SHA"
+          ./trusted/maintainers/scripts/check-cherry-picks.sh "$BASE_SHA" "$HEAD_SHA"

.github/workflows/check-format.yml

@@ -16,13 +16,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
         with:
@@ -33,7 +30,7 @@ jobs:
           # Note that it's fine to run this on untrusted code because:
           # - There's no secrets accessible here
           # - The build is sandboxed
-          if ! nix-build ci -A fmt.check; then
+          if ! nix-build untrusted/ci -A fmt.check; then
             echo "Some files are not properly formatted"
             echo "Please format them by going to the Nixpkgs root directory and running one of:"
             echo "  nix-shell --run treefmt"

.github/workflows/check-shell.yml

@@ -33,15 +33,12 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
 
       - name: Build shell
-        run: nix-build ci -A shell
+        run: nix-build untrusted/ci -A shell

.github/workflows/codeowners-v2.yml

@@ -46,9 +46,11 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge and target commits
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
+        with:
+          merged-as-untrusted: true
+          target-as-trusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
 
@@ -58,15 +60,8 @@ jobs:
           name: nixpkgs-ci
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
-      # Important: Because we use pull_request_target, this checks out the base branch of the PR, not the PR itself.
-      # We later build and run code from the base branch with access to secrets,
-      # so it's important this is not the PRs code.
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          path: base
-
       - name: Build codeowners validator
-        run: nix-build base/ci -A codeownersValidator
+        run: nix-build trusted/ci -A codeownersValidator
 
       - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
         if: vars.OWNER_RO_APP_ID
@@ -77,17 +72,12 @@ jobs:
           permission-administration: read
           permission-members: read
 
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
-          path: pr
-
       - name: Validate codeowners
         if: steps.app-token.outputs.token
         env:
-          OWNERS_FILE: pr/${{ env.OWNERS_FILE }}
+          OWNERS_FILE: untrusted/${{ env.OWNERS_FILE }}
           GITHUB_ACCESS_TOKEN: ${{ steps.app-token.outputs.token }}
-          REPOSITORY_PATH: pr
+          REPOSITORY_PATH: untrusted
           OWNER_CHECKER_REPOSITORY: ${{ github.repository }}
           # Set this to "notowned,avoid-shadowing" to check that all files are owned by somebody
           EXPERIMENTAL_CHECKS: "avoid-shadowing"
@@ -104,6 +94,8 @@ jobs:
       # Important: Because we use pull_request_target, this checks out the base branch of the PR, not the PR head.
       # This is intentional, because we need to request the review of owners as declared in the base branch.
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          path: trusted
 
       - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
         if: vars.OWNER_APP_ID
@@ -116,7 +108,7 @@ jobs:
           permission-pull-requests: write
 
       - name: Build review request package
-        run: nix-build ci -A requestReviews
+        run: nix-build trusted/ci -A requestReviews
 
       - name: Request reviews
         if: steps.app-token.outputs.token

.github/workflows/eval-aliases.yml

@@ -16,15 +16,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - name: Check out the PR at the test merge commit
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
-          path: nixpkgs
+          merged-as-untrusted: true
 
       - name: Install Nix
         uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
@@ -32,8 +27,8 @@ jobs:
           extra_nix_config: sandbox = true
 
       - name: Ensure flake outputs on all systems still evaluate
-        run: nix flake check --all-systems --no-build ./nixpkgs
+        run: nix flake check --all-systems --no-build ./untrusted
 
       - name: Query nixpkgs with aliases enabled to check for basic syntax errors
         run: |
-          time nix-env -I ./nixpkgs -f ./nixpkgs -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
+          time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null

.github/workflows/eval.yml

@@ -61,7 +61,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ needs.prepare.outputs.mergedSha }}
-          path: nixpkgs
+          path: untrusted
 
       - name: Install Nix
         uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
@@ -72,7 +72,7 @@ jobs:
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
         run: |
-          nix-build nixpkgs/ci -A eval.singleSystem \
+          nix-build untrusted/ci -A eval.singleSystem \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --arg chunkSize 10000
           # If it uses too much memory, slightly decrease chunkSize
@@ -101,8 +101,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ needs.prepare.outputs.mergedSha }}
-          fetch-depth: 2
-          path: nixpkgs
+          path: untrusted
 
       - name: Install Nix
         uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
@@ -111,7 +110,7 @@ jobs:
 
       - name: Combine all output paths and eval stats
         run: |
-          nix-build nixpkgs/ci -A eval.combine \
+          nix-build untrusted/ci -A eval.combine \
             --arg resultsDir ./intermediate \
             -o prResult
 
@@ -168,12 +167,13 @@ jobs:
         env:
           AUTHOR_ID: ${{ github.event.pull_request.user.id }}
         run: |
-          git -C nixpkgs worktree add ../target ${{ needs.prepare.outputs.targetSha }}
-          git -C nixpkgs diff --name-only ${{ needs.prepare.outputs.targetSha }} \
+          git -C untrusted fetch --depth 1 origin ${{ needs.prepare.outputs.targetSha }}
+          git -C untrusted worktree add ../trusted ${{ needs.prepare.outputs.targetSha }}
+          git -C untrusted diff --name-only ${{ needs.prepare.outputs.targetSha }} \
             | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
 
           # Use the target branch to get accurate maintainer info
-          nix-build target/ci -A eval.compare \
+          nix-build trusted/ci -A eval.compare \
             --arg beforeResultDir ./targetResult \
             --arg afterResultDir "$(realpath prResult)" \
             --arg touchedFilesJson ./touched-files.json \
@@ -222,15 +222,15 @@ jobs:
 
       # Important: This workflow job runs with extra permissions,
       # so we need to make sure to not run untrusted code from PRs
-      - name: Check out Nixpkgs at the base commit
+      - name: Check out Nixpkgs at the target commit
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ needs.prepare.outputs.targetSha }}
-          path: base
+          path: trusted
           sparse-checkout: ci
 
       - name: Build the requestReviews derivation
-        run: nix-build base/ci -A requestReviews
+        run: nix-build trusted/ci -A requestReviews
 
       - name: Labelling pull request
         if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}

.github/workflows/lib-tests.yml

@@ -19,13 +19,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
         with:
@@ -33,4 +30,4 @@ jobs:
 
       - name: Building Nixpkgs lib-tests
         run: |
-          nix-build ci -A lib-tests
+          nix-build untrusted/ci -A lib-tests

.github/workflows/manual-nixos-v2.yml

@@ -35,13 +35,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
         with:
@@ -55,7 +52,7 @@ jobs:
 
       - name: Build NixOS manual
         id: build-manual
-        run: NIX_PATH=nixpkgs=$(pwd) nix-build --option restrict-eval true ci -A manual-nixos --argstr system ${{ matrix.system }}
+        run: NIX_PATH=nixpkgs=$(pwd)/untrusted nix-build --option restrict-eval true untrusted/ci -A manual-nixos --argstr system ${{ matrix.system }}
 
       - name: Upload NixOS manual
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

.github/workflows/manual-nixpkgs-v2.yml

@@ -22,13 +22,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
         with:
@@ -41,4 +38,4 @@ jobs:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
       - name: Building Nixpkgs manual
-        run: NIX_PATH=nixpkgs=$(pwd) nix-build --option restrict-eval true ci -A manual-nixpkgs -A manual-nixpkgs-tests
+        run: NIX_PATH=nixpkgs=$(pwd)/untrusted nix-build --option restrict-eval true untrusted/ci -A manual-nixpkgs -A manual-nixpkgs-tests

.github/workflows/nix-parse-v2.yml

@@ -17,13 +17,10 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout the merge commit
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
+          merged-as-untrusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
         with:
@@ -33,4 +30,4 @@ jobs:
       - name: Parse all nix files
         run: |
           # Tests multiple versions at once, let's make sure all of them run, so keep-going.
-          nix-build ci -A parse --keep-going
+          nix-build untrusted/ci -A parse --keep-going

.github/workflows/nixpkgs-vet.yml

@@ -19,51 +19,27 @@ permissions: {}
 jobs:
   check:
     name: nixpkgs-vet
-    # This needs to be x86_64-linux, because we depend on the tooling being pre-built in the GitHub releases.
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-24.04-arm
     # This should take 1 minute at most, but let's be generous. The default of 6 hours is definitely too long.
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/actions
-      - name: Check if the PR can be merged and get the test merge commit
+      - name: Check if the PR can be merged and checkout merged and target commits
         uses: ./.github/actions/get-merge-commit
-        id: get-merge-commit
-
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: ${{ steps.get-merge-commit.outputs.mergedSha }}
-          # Fetches the merge commit and its parents
-          fetch-depth: 2
-
-      - name: Checking out target branch
-        run: |
-          target=$(mktemp -d)
-          git worktree add "$target" "$(git rev-parse HEAD^1)"
-          echo "target=$target" >> "$GITHUB_ENV"
+          merged-as-untrusted: true
+          target-as-trusted: true
 
       - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
 
-      - name: Fetching the pinned tool
-        # Update the pinned version using ci/nixpkgs-vet/update-pinned-tool.sh
-        run: |
-          # The pinned version of the tooling to use.
-          toolVersion=$(<ci/nixpkgs-vet/pinned-version.txt)
-
-          # Fetch the x86_64-linux-specific release artifact containing the gzipped NAR of the pre-built tool.
-          toolPath=$(curl -sSfL https://github.com/NixOS/nixpkgs-vet/releases/download/"$toolVersion"/x86_64-linux.nar.gz \
-            | gzip -cd | nix-store --import | tail -1)
-
-          # Adds a result symlink as a GC root.
-          nix-store --realise "$toolPath" --add-root result
-
       - name: Running nixpkgs-vet
         env:
           # Force terminal colors to be enabled. The library that `nixpkgs-vet` uses respects https://bixense.com/clicolors/
           CLICOLOR_FORCE: 1
         run: |
-          if result/bin/nixpkgs-vet --base "$target" .; then
+          if nix-build untrusted/ci -A nixpkgs-vet --arg base "./trusted" --arg head "./untrusted"; then
             exit 0
           else
             exitCode=$?

ci/default.nix

@@ -84,6 +84,7 @@ in
   manual-nixos = (import ../nixos/release.nix { }).manual.${system} or null;
   manual-nixpkgs = (import ../pkgs/top-level/release.nix { }).manual;
   manual-nixpkgs-tests = (import ../pkgs/top-level/release.nix { }).manual.tests;
+  nixpkgs-vet = pkgs.callPackage ./nixpkgs-vet.nix { };
   parse = pkgs.lib.recurseIntoAttrs {
     latest = pkgs.callPackage ./parse.nix { nix = pkgs.nixVersions.latest; };
     lix = pkgs.callPackage ./parse.nix { nix = pkgs.lix; };

ci/nixpkgs-vet.nix

@@ -0,0 +1,31 @@
+{
+  lib,
+  nix,
+  nixpkgs-vet,
+  runCommand,
+}:
+{
+  base ? ../.,
+  head ? ../.,
+}:
+let
+  filtered =
+    with lib.fileset;
+    path:
+    toSource {
+      fileset = (gitTracked path);
+      root = path;
+    };
+in
+runCommand "nixpkgs-vet"
+  {
+    nativeBuildInputs = [
+      nixpkgs-vet
+    ];
+    env.NIXPKGS_VET_NIX_PACKAGE = nix;
+  }
+  ''
+    nixpkgs-vet --base ${filtered base} ${filtered head}
+
+    touch $out
+  ''

ci/nixpkgs-vet.sh

@@ -65,7 +65,5 @@ trace -n "Reading pinned nixpkgs-vet version from pinned-version.txt.. "
 toolVersion=$(<"$tmp/merged/ci/nixpkgs-vet/pinned-version.txt")
 trace -e "\e[34m$toolVersion\e[0m"
 
-trace -n "Building tool.. "
-nix-build https://github.com/NixOS/nixpkgs-vet/tarball/"$toolVersion" -o "$tmp/tool" -A build
 trace "Running nixpkgs-vet.."
-"$tmp/tool/bin/nixpkgs-vet" --base "$tmp/base" "$tmp/merged"
+nix-build ci -A nixpkgs-vet --argstr base "$tmp/base" --argstr head "$tmp/merged"