From 5937bc341dbf885a2034ee9e4e75ab3cf48c3e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 23 Aug 2026 07:02:39 +0200 Subject: [PATCH 1/4] top-level/release.nix: use named constituents in aggregates The unstable and darwin-tested aggregates still reference their constituents as derivations. This forces the evaluation of ~47 jobs plus all stdenv bootstrap-tool sets into the single evaluator worker that evals the aggregate. On top of evaluating them again as their own jobs. The NixOS jobsets switched to named constituents a while ago. Hydra resolves the strings after evaluation and rewrites the aggregate to depend on the resolved builds, so channel gating works as before. Evaluating just the unstable aggregate drops from 1.5 GB / 19 s to 77 MB / 0.4 s: nix-instantiate --eval --strict -A unstable.drvPath pkgs/top-level/release.nix Note that nix-build -A unstable no longer builds the constituents, same as NixOS jobsets. --- pkgs/top-level/release.nix | 172 +++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 85 deletions(-) diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index a6b22fba574e..b277da8b043a 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -77,13 +77,12 @@ let inherit (release-lib) mapTestOn pkgs; inherit (release-lib.lib) - collect + concatMap elem genAttrs hasInfix hasSuffix id - isDerivation optionals ; @@ -113,49 +112,49 @@ let name = "nixpkgs-darwin-${jobs.tarball.version}"; meta.description = "Release-critical builds for the Nixpkgs darwin channel"; constituents = [ - jobs.tarball - jobs.release-checks + "tarball" + "release-checks" ] ++ optionals supportDarwin [ - jobs.cabal2nix.aarch64-darwin - jobs.ghc.aarch64-darwin - jobs.git.aarch64-darwin - jobs.go.aarch64-darwin - jobs.mariadb.aarch64-darwin - jobs.nix.aarch64-darwin - jobs.nixpkgs-review.aarch64-darwin - jobs.nix-info.aarch64-darwin - jobs.nix-info-tested.aarch64-darwin - jobs.openssh.aarch64-darwin - jobs.openssl.aarch64-darwin - jobs.pandoc.aarch64-darwin - jobs.postgresql.aarch64-darwin - jobs.python3.aarch64-darwin - jobs.ruby.aarch64-darwin - jobs.rustc.aarch64-darwin + "cabal2nix.aarch64-darwin" + "ghc.aarch64-darwin" + "git.aarch64-darwin" + "go.aarch64-darwin" + "mariadb.aarch64-darwin" + "nix.aarch64-darwin" + "nixpkgs-review.aarch64-darwin" + "nix-info.aarch64-darwin" + "nix-info-tested.aarch64-darwin" + "openssh.aarch64-darwin" + "openssl.aarch64-darwin" + "pandoc.aarch64-darwin" + "postgresql.aarch64-darwin" + "python3.aarch64-darwin" + "ruby.aarch64-darwin" + "rustc.aarch64-darwin" # blocking ofBorg CI 2020-02-28 - # jobs.stack.aarch64-darwin - jobs.stdenv.aarch64-darwin - jobs.vim.aarch64-darwin - jobs.cachix.aarch64-darwin - jobs.darwin.linux-builder.aarch64-darwin + # "stack.aarch64-darwin" + "stdenv.aarch64-darwin" + "vim.aarch64-darwin" + "cachix.aarch64-darwin" + "darwin.linux-builder.aarch64-darwin" # UI apps - # jobs.firefox-unwrapped.aarch64-darwin - jobs.qt5.qtmultimedia.aarch64-darwin - jobs.inkscape.aarch64-darwin - jobs.gimp2.aarch64-darwin # FIXME replace with gimp once https://github.com/NixOS/nixpkgs/issues/411189 is resolved - jobs.emacs.aarch64-darwin - jobs.wireshark.aarch64-darwin + # "firefox-unwrapped.aarch64-darwin" + "qt5.qtmultimedia.aarch64-darwin" + "inkscape.aarch64-darwin" + "gimp2.aarch64-darwin" # FIXME replace with gimp once https://github.com/NixOS/nixpkgs/issues/411189 is resolved + "emacs.aarch64-darwin" + "wireshark.aarch64-darwin" # Tests /* - jobs.tests.cc-wrapper.default.aarch64-darwin - jobs.tests.cc-wrapper.llvmPackages.clang.aarch64-darwin - jobs.tests.cc-wrapper.llvmPackages.libcxx.aarch64-darwin - jobs.tests.stdenv-inputs.aarch64-darwin - jobs.tests.macOSSierraShared.aarch64-darwin - jobs.tests.stdenv.hooks.patch-shebangs.aarch64-darwin + "tests.cc-wrapper.default.aarch64-darwin" + "tests.cc-wrapper.llvmPackages.clang.aarch64-darwin" + "tests.cc-wrapper.llvmPackages.libcxx.aarch64-darwin" + "tests.stdenv-inputs.aarch64-darwin" + "tests.macOSSierraShared.aarch64-darwin" + "tests.stdenv.hooks.patch-shebangs.aarch64-darwin" */ ]; } @@ -166,68 +165,71 @@ let name = "nixpkgs-${jobs.tarball.version}"; meta.description = "Release-critical builds for the Nixpkgs unstable channel"; constituents = [ - jobs.tarball - jobs.release-checks - jobs.metrics - jobs.manual - jobs.tests.lib-tests.x86_64-linux - jobs.tests.pkgs-lib.formats-tests.x86_64-linux - jobs.stdenv.x86_64-linux - jobs.cargo.x86_64-linux - jobs.go.x86_64-linux - jobs.linux.x86_64-linux - jobs.nix.x86_64-linux - jobs.pandoc.x86_64-linux - jobs.python3.x86_64-linux + "tarball" + "release-checks" + "metrics" + "manual" + "tests.lib-tests.x86_64-linux" + "tests.pkgs-lib.formats-tests.x86_64-linux" + "stdenv.x86_64-linux" + "cargo.x86_64-linux" + "go.x86_64-linux" + "linux.x86_64-linux" + "nix.x86_64-linux" + "pandoc.x86_64-linux" + "python3.x86_64-linux" # Needed by contributors to test PRs (by inclusion of the PR template) - jobs.nixpkgs-review.x86_64-linux + "nixpkgs-review.x86_64-linux" # Needed for support - jobs.nix-info.x86_64-linux - jobs.nix-info-tested.x86_64-linux + "nix-info.x86_64-linux" + "nix-info-tested.x86_64-linux" # Ensure that X11/GTK are in order. - jobs.firefox-unwrapped.x86_64-linux - jobs.cachix.x86_64-linux - jobs.devenv.x86_64-linux + "firefox-unwrapped.x86_64-linux" + "cachix.x86_64-linux" + "devenv.x86_64-linux" /* TODO: re-add tests; context: https://github.com/NixOS/nixpkgs/commit/36587a587ab191eddd868179d63c82cdd5dee21b - jobs.tests.cc-wrapper.default.x86_64-linux + "tests.cc-wrapper.default.x86_64-linux" # broken see issue #40038 - jobs.tests.cc-wrapper.llvmPackages.clang.x86_64-linux - jobs.tests.cc-wrapper.llvmPackages.libcxx.x86_64-linux - jobs.tests.cc-multilib-gcc.x86_64-linux - jobs.tests.cc-multilib-clang.x86_64-linux - jobs.tests.stdenv-inputs.x86_64-linux - jobs.tests.stdenv.hooks.patch-shebangs.x86_64-linux + "tests.cc-wrapper.llvmPackages.clang.x86_64-linux" + "tests.cc-wrapper.llvmPackages.libcxx.x86_64-linux" + "tests.cc-multilib-gcc.x86_64-linux" + "tests.cc-multilib-clang.x86_64-linux" + "tests.stdenv-inputs.x86_64-linux" + "tests.stdenv.hooks.patch-shebangs.x86_64-linux" */ ] - ++ collect isDerivation jobs.stdenvBootstrapTools + ++ concatMap (config: [ + "stdenvBootstrapTools.${config}.build" + "stdenvBootstrapTools.${config}.test" + ]) bootstrapConfigs ++ optionals supportDarwin [ - jobs.stdenv.aarch64-darwin - jobs.cargo.aarch64-darwin - jobs.cachix.aarch64-darwin - jobs.devenv.aarch64-darwin - jobs.go.aarch64-darwin - jobs.python3.aarch64-darwin - jobs.nixpkgs-review.aarch64-darwin - jobs.nix.aarch64-darwin - jobs.nix-info.aarch64-darwin - jobs.nix-info-tested.aarch64-darwin - jobs.git.aarch64-darwin - jobs.mariadb.aarch64-darwin - jobs.vim.aarch64-darwin - jobs.inkscape.aarch64-darwin - jobs.qt5.qtmultimedia.aarch64-darwin - jobs.darwin.linux-builder.aarch64-darwin + "stdenv.aarch64-darwin" + "cargo.aarch64-darwin" + "cachix.aarch64-darwin" + "devenv.aarch64-darwin" + "go.aarch64-darwin" + "python3.aarch64-darwin" + "nixpkgs-review.aarch64-darwin" + "nix.aarch64-darwin" + "nix-info.aarch64-darwin" + "nix-info-tested.aarch64-darwin" + "git.aarch64-darwin" + "mariadb.aarch64-darwin" + "vim.aarch64-darwin" + "inkscape.aarch64-darwin" + "qt5.qtmultimedia.aarch64-darwin" + "darwin.linux-builder.aarch64-darwin" /* - jobs.tests.cc-wrapper.default.aarch64-darwin - jobs.tests.cc-wrapper.llvmPackages.clang.aarch64-darwin - jobs.tests.cc-wrapper.llvmPackages.libcxx.aarch64-darwin - jobs.tests.stdenv-inputs.aarch64-darwin - jobs.tests.stdenv.hooks.patch-shebangs.aarch64-darwin + "tests.cc-wrapper.default.aarch64-darwin" + "tests.cc-wrapper.llvmPackages.clang.aarch64-darwin" + "tests.cc-wrapper.llvmPackages.libcxx.aarch64-darwin" + "tests.stdenv-inputs.aarch64-darwin" + "tests.stdenv.hooks.patch-shebangs.aarch64-darwin" */ ]; }; From 36f7e3e5de193ae9660a9bedef318c8110d344eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 23 Aug 2026 07:27:24 +0200 Subject: [PATCH 2/4] releaseTools.aggregate: support glob constituents Lets jobsets reference whole job subtrees by fnmatch pattern instead of forcing their evaluation inside the aggregate. --- pkgs/build-support/release/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/build-support/release/default.nix b/pkgs/build-support/release/default.nix index 6ad860d4fac9..2ac928d77916 100644 --- a/pkgs/build-support/release/default.nix +++ b/pkgs/build-support/release/default.nix @@ -124,12 +124,16 @@ rec { name, constituents, meta ? { }, + # Interpret string constituents as fnmatch patterns matched + # against all job names of the jobset. + globConstituents ? false, }: pkgs.runCommand name { inherit constituents meta; preferLocalBuild = true; _hydraAggregate = true; + _hydraGlobConstituents = globConstituents; } '' mkdir -p $out/nix-support From 252f737b2610b23fecc514aeaa622a4f7a55c4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 23 Aug 2026 07:27:24 +0200 Subject: [PATCH 3/4] release-haskell.nix: use glob constituents in aggregates The aggregates evaluated every constituent twice, once inside a single evaluator worker and once as its own job. Evaluating the mergeable aggregate alone drops from 1.3 GB / 15 s to 73 MB / 0.4 s. --- pkgs/top-level/release-haskell.nix | 86 ++++++++++++------------------ 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/pkgs/top-level/release-haskell.nix b/pkgs/top-level/release-haskell.nix index 686d09922259..fe773844c1e8 100644 --- a/pkgs/top-level/release-haskell.nix +++ b/pkgs/top-level/release-haskell.nix @@ -26,26 +26,6 @@ let pkgs ; - # Helper function which traverses a (nested) set - # of derivations produced by mapTestOn and flattens - # it to a list of derivations suitable to be passed - # to `releaseTools.aggregate` as constituents. - # Removes all non derivations from the input jobList. - # - # accumulateDerivations :: [ Either Derivation AttrSet ] -> [ Derivation ] - # - # > accumulateDerivations [ drv1 "string" { foo = drv2; bar = { baz = drv3; }; } ] - # [ drv1 drv2 drv3 ] - accumulateDerivations = - jobList: - lib.concatMap ( - attrs: - if lib.isDerivation attrs then - [ attrs ] - else - lib.optionals (lib.isAttrs attrs) (accumulateDerivations (lib.attrValues attrs)) - ) jobList; - # names of all subsets of `pkgs.haskell.packages` # # compilerNames looks like the following: @@ -632,34 +612,35 @@ let ''; teams = [ lib.teams.haskell ]; }; - constituents = accumulateDerivations [ + globConstituents = true; + constituents = [ # haskell specific tests - jobs.tests.haskell + "tests.haskell.*" # important top-level packages - jobs.cabal-install - jobs.cabal2nix - jobs.cachix - jobs.darcs - jobs.haskell-language-server - jobs.hledger - jobs.hledger-ui - jobs.hpack - jobs.niv - jobs.pandoc - jobs.stack - jobs.stylish-haskell - jobs.shellcheck + "cabal-install.*" + "cabal2nix.*" + "cachix.*" + "darcs.*" + "haskell-language-server.*" + "hledger.*" + "hledger-ui.*" + "hpack.*" + "niv.*" + "pandoc.*" + "stack.*" + "stylish-haskell.*" + "shellcheck.*" # important haskell (library) packages - jobs.haskellPackages.cabal-plan - jobs.haskellPackages.distribution-nixpkgs - jobs.haskellPackages.hackage-db - jobs.haskellPackages.xmonad - jobs.haskellPackages.xmonad-contrib + "haskellPackages.cabal-plan.*" + "haskellPackages.distribution-nixpkgs.*" + "haskellPackages.hackage-db.*" + "haskellPackages.xmonad.*" + "haskellPackages.xmonad-contrib.*" # haskell packages maintained by @peti # imported from the old hydra jobset - jobs.haskellPackages.hopenssl - jobs.haskellPackages.hsemail - jobs.haskellPackages.hsyslog + "haskellPackages.hopenssl.*" + "haskellPackages.hsemail.*" + "haskellPackages.hsyslog.*" ]; }; maintained = pkgs.releaseTools.aggregate { @@ -668,9 +649,8 @@ let description = "Aggregate jobset of all haskell packages with a maintainer"; teams = [ lib.teams.haskell ]; }; - constituents = accumulateDerivations ( - map (name: jobs.haskellPackages."${name}") (maintainedPkgNames pkgs.haskellPackages) - ); + globConstituents = true; + constituents = map (name: "haskellPackages.${name}.*") (maintainedPkgNames pkgs.haskellPackages); }; muslGHCs = pkgs.releaseTools.aggregate { @@ -681,9 +661,10 @@ let nh2 ]; }; - constituents = accumulateDerivations [ - jobs.pkgsMusl.haskell.compiler.ghcHEAD - jobs.pkgsMusl.haskell.compiler.native-bignum.ghcHEAD + globConstituents = true; + constituents = [ + "pkgsMusl.haskell.compiler.ghcHEAD.*" + "pkgsMusl.haskell.compiler.native-bignum.ghcHEAD.*" ]; }; @@ -696,9 +677,10 @@ let lib.maintainers.rnhmjoj ]; }; - constituents = accumulateDerivations [ - jobs.pkgsStatic.haskell.packages.native-bignum.ghc948 # non-hadrian - jobs.pkgsStatic.haskellPackages + globConstituents = true; + constituents = [ + "pkgsStatic.haskell.packages.native-bignum.ghc948.*" # non-hadrian + "pkgsStatic.haskellPackages.*" ]; }; } From 9305dec609e47f19923528af180ed1dd0dde0ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 23 Aug 2026 07:27:24 +0200 Subject: [PATCH 4/4] release-python.nix: use named constituents in the aggregate Same change as for the other jobsets. --- pkgs/top-level/release-python.nix | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkgs/top-level/release-python.nix b/pkgs/top-level/release-python.nix index 51407db2654f..51faf69aec8b 100644 --- a/pkgs/top-level/release-python.nix +++ b/pkgs/top-level/release-python.nix @@ -53,14 +53,14 @@ let name = "python-tested"; meta.description = "Release-critical packages from the python package sets"; constituents = [ - jobs.nixos-render-docs.x86_64-linux # Used in nixos manual - jobs.remarshal.x86_64-linux # Used in pkgs.formats.yaml_1_1 - jobs.python313Packages.afdko.x86_64-linux # Used in noto-fonts-color-emoji - jobs.python313Packages.buildcatrust.x86_64-linux # Used in pkgs.cacert - jobs.python313Packages.colorama.x86_64-linux # Used in nixos test-driver - jobs.python313Packages.ptpython.x86_64-linux # Used in nixos test-driver - jobs.python313Packages.requests.x86_64-linux # Almost ubiquous package - jobs.python313Packages.sphinx.x86_64-linux # Document creation for many packages + "nixos-render-docs.x86_64-linux" # Used in nixos manual + "remarshal.x86_64-linux" # Used in pkgs.formats.yaml_1_1 + "python313Packages.afdko.x86_64-linux" # Used in noto-fonts-color-emoji + "python313Packages.buildcatrust.x86_64-linux" # Used in pkgs.cacert + "python313Packages.colorama.x86_64-linux" # Used in nixos test-driver + "python313Packages.ptpython.x86_64-linux" # Used in nixos test-driver + "python313Packages.requests.x86_64-linux" # Almost ubiquous package + "python313Packages.sphinx.x86_64-linux" # Document creation for many packages ]; };