diff --git a/doc/hooks/check-phase-thread-limit-hook.section.md b/doc/hooks/check-phase-thread-limit-hook.section.md new file mode 100644 index 000000000000..0adc0d309c47 --- /dev/null +++ b/doc/hooks/check-phase-thread-limit-hook.section.md @@ -0,0 +1,24 @@ +# checkPhaseThreadLimitHook {#setup-hook-check-phase-thread-limit} + +This hook defaults a variety of environment variables known +to control thread counts to 1. Many of these otherwise default +to `$(nproc)`, which causes massive overloads on build machines +if nix build jobs and build cores are already tuned to fully utilize +compute capacity of a builder without additional parallelism. + +Currently sets the following environment variables: +- [`OMP_NUM_THREADS`](https://www.openmp.org/spec-html/5.0/openmpse50.html) +- [`OPENBLAS_NUM_THREADS`](https://github.com/OpenMathLib/OpenBLAS/blob/e7b45174355edec1f04de1cabcf5ca6a98ea7fbc/USAGE.md#how-can-i-use-openblas-in-multi-threaded-applications) +- [`MKL_NUM_THREADS`](https://www.intel.com/content/www/us/en/docs/onemkl/developer-guide-linux/2023-0/mkl-domain-num-threads.html) +- [`BLIS_NUM_THREADS`](https://github.com/flame/blis/blob/b8b75b4e19459f5d618b57aa814ca38b1d82eb82/docs/Multithreading.md#specifying-multithreading) +- `VECLIB_MAXIMUM_THREADS`: Only affects darwin, see [`man 7 Accelerate`](https://manp.gs/mac/7/Accelerate) +- [`NUMBA_NUM_THREADS`](https://numba.readthedocs.io/en/stable/reference/envvars.html#threading-control) +- [`NUMEXPR_NUM_THREADS`](https://numexpr.readthedocs.io/en/latest/user_guide.html#threadpool-configuration) + +The `NIX_CHECK_PHASE_DEFAULT_NUM_THREADS` environment variable +can be used to override the default thread count limit. +`dontLimitCheckPhaseThreads = true;` can be used to disable +thread limiting on an individual package. + +This hook will not attempt to override already existing +definitions for thread count environment variables. diff --git a/doc/hooks/index.md b/doc/hooks/index.md index e6c3af542f07..d6a470f506e3 100644 --- a/doc/hooks/index.md +++ b/doc/hooks/index.md @@ -13,6 +13,7 @@ aws-c-common.section.md bmake.section.md breakpoint.section.md cernlib.section.md +check-phase-thread-limit-hook.section.md cmake.section.md desktop-file-utils.section.md gdk-pixbuf.section.md @@ -34,7 +35,6 @@ nodejs-install-manuals.section.md npm-build-hook.section.md npm-config-hook.section.md npm-install-hook.section.md -openmp-check-hook.section.md patch-rc-path-hooks.section.md perl.section.md pkg-config.section.md diff --git a/doc/hooks/openmp-check-hook.section.md b/doc/hooks/openmp-check-hook.section.md deleted file mode 100644 index 5a5842eb5e29..000000000000 --- a/doc/hooks/openmp-check-hook.section.md +++ /dev/null @@ -1,10 +0,0 @@ -# openmpCheckPhaseHook {#setup-hook-omp-check} - - -This hook can be used to setup a check phase that -requires running a OpenMP application. It mostly -serves to limit `OMP_NUM_THREADS` to avoid overloading -build machines. - -This hook will not attempt to override an already existing -definition of `OMP_NUM_THREADS` in the environment. diff --git a/doc/redirects.json b/doc/redirects.json index 8a822e998401..edc4696b1821 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -2809,7 +2809,8 @@ "setup-hook-mpi-check": [ "index.html#setup-hook-mpi-check" ], - "setup-hook-omp-check": [ + "setup-hook-check-phase-thread-limit": [ + "index.html#setup-hook-check-phase-thread-limit", "index.html#setup-hook-omp-check" ], "ninja": [ diff --git a/pkgs/by-name/ch/checkPhaseThreadLimitHook/hook.sh b/pkgs/by-name/ch/checkPhaseThreadLimitHook/hook.sh new file mode 100644 index 000000000000..1b610b631d84 --- /dev/null +++ b/pkgs/by-name/ch/checkPhaseThreadLimitHook/hook.sh @@ -0,0 +1,32 @@ +setupThreadLimit() { + # Limit number of threads used during checks. Default is "all cores". + # Using all cores causes high load on builders if checks are executed with NIX_BUILD_CORE parallelism. + # This gets even worse if multiple builds are scheduled on the same machine, potentially growing O(n^3) without explicit core limits. + + # global value to override all of these at once + NIX_CHECK_PHASE_DEFAULT_NUM_THREADS="${NIX_CHECK_PHASE_DEFAULT_NUM_THREADS:-1}" + + thread_vars=( + OMP_NUM_THREADS + OPENBLAS_NUM_THREADS + MKL_NUM_THREADS + BLIS_NUM_THREADS + VECLIB_MAXIMUM_THREADS + NUMBA_NUM_THREADS + NUMEXPR_NUM_THREADS + ) + + echo "setting known thread variables to default: $NIX_CHECK_PHASE_DEFAULT_NUM_THREADS" + for var in "${thread_vars[@]}"; do + export "$var=${!var:-$NIX_CHECK_PHASE_DEFAULT_NUM_THREADS}" + printf " %s=%s" "$var" "${!var}" + done + printf "\n" +} + + +if [[ -z "${dontLimitCheckPhaseThreads-}" ]]; then + echo "Using checkPhaseThreadLimitHook" + preCheckHooks+=('setupThreadLimit') + preInstallCheckHooks+=('setupThreadLimit') +fi diff --git a/pkgs/by-name/ch/checkPhaseThreadLimitHook/package.nix b/pkgs/by-name/ch/checkPhaseThreadLimitHook/package.nix new file mode 100644 index 000000000000..6bd9bda7135a --- /dev/null +++ b/pkgs/by-name/ch/checkPhaseThreadLimitHook/package.nix @@ -0,0 +1,15 @@ +{ + lib, + makeSetupHook, +}: + +makeSetupHook { + name = "check-phase-thread-limit-hook"; + + __structuredAttrs = true; + + meta = { + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ grimmauld ]; + }; +} ./hook.sh diff --git a/pkgs/by-name/op/openmpCheckPhaseHook/omp-check-hook.sh b/pkgs/by-name/op/openmpCheckPhaseHook/omp-check-hook.sh deleted file mode 100644 index b3279b170360..000000000000 --- a/pkgs/by-name/op/openmpCheckPhaseHook/omp-check-hook.sh +++ /dev/null @@ -1,11 +0,0 @@ -preCheckHooks+=('setupOmpCheck') -preInstallCheckHooks+=('setupOmpCheck') - - -setupOmpCheck() { - # Limit number of OpenMP threads. Default is "all cores". - # Using all cores causes high load on builders if checks are executed with NIX_BUILD_CORE parallelism. - # This gets even worse if multiple builds are scheduled on the same machine, potentially growing O(n^3) without explicit core limits. - export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" -} - diff --git a/pkgs/by-name/op/openmpCheckPhaseHook/package.nix b/pkgs/by-name/op/openmpCheckPhaseHook/package.nix deleted file mode 100644 index 9e30609e8de5..000000000000 --- a/pkgs/by-name/op/openmpCheckPhaseHook/package.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - lib, - makeSetupHook, -}: - -makeSetupHook { - name = "omp-checkPhase-hook"; - - __structuredAttrs = true; - - meta.license = lib.licenses.mit; -} ./omp-check-hook.sh diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index cabc64941662..3d1006cb06d2 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -1629,6 +1629,7 @@ mapAliases { openjfx23 = throw "OpenJFX 23 was removed as it has reached its end of life"; # Added 2025-11-04 openjfx24 = throw "OpenJFX 24 was removed as it has reached its end of life"; # Added 2025-10-04 openmodelica = throw "'openmodelica' has been removed as it was unmaintained in nixpkgs and depends on insecure&unmtaintained qtwebkit"; # Added 2026-04-26 + openmpCheckPhaseHook = warnAlias "'openmpCheckPhaseHook' has been renamed to 'checkPhaseThreadLimitHook' to reflect its handling of all known thread-limiting mechanisms during check phase" checkPhaseThreadLimitHook; # Added 2026-07-09 openmw-tes3mp = throw "'openmw-tes3mp' has been removed due to lack of maintenance upstream"; # Added 2025-08-30 openssl_3_0 = throw "'openssl_3_0' has been renamed to/replaced by 'openssl_3'"; # Converted to throw 2025-10-27 opensycl = throw "'opensycl' has been renamed to/replaced by 'adaptivecpp'"; # Converted to throw 2025-10-27