diff --git a/ci/eval/chunk.nix b/ci/eval/chunk.nix index e1759c2c6615..4fabea659ad7 100644 --- a/ci/eval/chunk.nix +++ b/ci/eval/chunk.nix @@ -2,8 +2,8 @@ { lib ? import ../../lib, path ? ../.., - # The file containing all available attribute paths, which are split into chunks here - attrpathFile, + # The file containing the preEval result + preEvalFile, chunkSize, myChunk, includeBroken, @@ -12,8 +12,8 @@ }: let - attrpaths = lib.importJSON attrpathFile; - myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths; + preEvalResult = lib.importJSON preEvalFile; + myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize preEvalResult.paths; unfiltered = import ./outpaths.nix { inherit path; diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 2e8bec41ca5d..547fcf1f26be 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -38,7 +38,7 @@ let fileset = unions ( map (lib.path.append ../..) [ ".version" - "ci/eval/attrpaths.nix" + "ci/eval/pre-eval.nix" "ci/eval/chunk.nix" "ci/eval/outpaths.nix" "default.nix" @@ -56,11 +56,11 @@ let builtins.readFile ../../pkgs/top-level/release-supported-systems.json ); - attrpathsSuperset = + preEval = { evalSystem, }: - runCommand "attrpaths-superset.json" + runCommand "pre-eval" { src = nixpkgs; # Don't depend on -dev outputs to reduce closure size for CI. @@ -73,15 +73,15 @@ let export NIX_STATE_DIR=$(mktemp -d) mkdir $out export GC_INITIAL_HEAP_SIZE=4g - command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \ + command time -f "Pre-eval done [%MKB max resident, %Es elapsed] %C" \ nix-instantiate --eval --strict --json --show-trace \ - "$src/ci/eval/attrpaths.nix" \ - -A paths \ + "$src/ci/eval/pre-eval.nix" \ + -A result \ -I "$src" \ --argstr extraNixpkgsConfigJson ${lib.escapeShellArg (builtins.toJSON extraNixpkgsConfig)} \ --option restrict-eval true \ --option allow-import-from-derivation false \ - --option eval-system "${evalSystem}" > $out/paths.json + --option eval-system "${evalSystem}" > $out/result.json ''; singleSystem = @@ -90,8 +90,8 @@ let # Note that this is intentionally not called `system`, # because `--argstr system` would only be passed to the ci/default.nix file! evalSystem ? builtins.currentSystem, - # The path to the `paths.json` file from `attrpathsSuperset` - attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json", + # The path to the `result.json` file from `preEval` + preEvalFile ? "${preEval { inherit evalSystem; }}/result.json", }: let singleChunk = writeShellScript "single-chunk" '' @@ -121,12 +121,12 @@ let --show-trace \ --arg chunkSize "$chunkSize" \ --arg myChunk "$myChunk" \ - --arg attrpathFile "${attrpathFile}" \ + --arg preEvalFile "${preEvalFile}" \ --arg systems "[ \"$system\" ]" \ --arg includeBroken ${lib.boolToString includeBroken} \ --argstr extraNixpkgsConfigJson ${lib.escapeShellArg (builtins.toJSON extraNixpkgsConfig)} \ -I ${nixpkgs} \ - -I ${attrpathFile} \ + -I ${preEvalFile} \ > "$outputDir/result/$myChunk" \ 2> "$outputDir/stderr/$myChunk" exitCode=$? @@ -164,7 +164,7 @@ let echo "System: $evalSystem" cores=$NIX_BUILD_CORES echo "Cores: $cores" - attrCount=$(jq length "${attrpathFile}") + attrCount=$(jq '.paths | length' "${preEvalFile}") echo "Attribute count: $attrCount" echo "Chunk size: $chunkSize" # Same as `attrCount / chunkSize` but rounded up @@ -316,7 +316,7 @@ let in { inherit - attrpathsSuperset + preEval singleSystem diff combine diff --git a/ci/eval/outpaths.nix b/ci/eval/outpaths.nix index afbce48f3321..6fcd22fc74a6 100755 --- a/ci/eval/outpaths.nix +++ b/ci/eval/outpaths.nix @@ -6,7 +6,7 @@ includeBroken ? true, # set this to false to exclude meta.broken packages from the output path ? ./../.., - # used by ./attrpaths.nix + # used by ./pre-eval.nix attrNamesOnly ? false, # Set this to `null` to build for builtins.currentSystem only diff --git a/ci/eval/attrpaths.nix b/ci/eval/pre-eval.nix similarity index 84% rename from ci/eval/attrpaths.nix rename to ci/eval/pre-eval.nix index ff1bdabbb350..2d5a092747ad 100644 --- a/ci/eval/attrpaths.nix +++ b/ci/eval/pre-eval.nix @@ -1,6 +1,5 @@ -# This expression will, as efficiently as possible, dump a -# *superset* of all attrpaths of derivations which might be -# part of a release on *any* platform. +# This file does a fast pre-evaluation of Nixpkgs to determine: +# - paths: A *superset* of all attrpaths of derivations which might be part of a release on *any* platform. # # This expression runs single-threaded under all current Nix # implementations, but much faster and with much less memory @@ -10,9 +9,9 @@ # $NUM_CORES batches and evaluate the outpaths separately for each # batch, in parallel. # -# To dump the attrnames: +# To dump the result: # -# nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names +# nix-instantiate --eval --strict --json ci/eval/pre-eval.nix -A result # { lib ? import (path + "/lib"), @@ -81,5 +80,9 @@ let in { + # TODO: Do we still need these? Probably not inherit paths names; + result = { + inherit paths; + }; }