diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index 925662592b04..9fb6ea662746 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -513,10 +513,10 @@ Changes can include workarounds or bug fixes to existing PNPM issues. ##### Version history {#javascript-pnpm-fetcherVersion-versionHistory} -Version 3 is the recommended value for new packages. Versions 1 and 2 are deprecated and scheduled for removal in the 26.11 release; existing packages must migrate. +Version 3 is the minimum supported value. Versions 1 and 2 were removed in the 26.11 release; packages that still use them fail to evaluate and must migrate to `fetcherVersion = 3` (or later) and regenerate their hashes. -- 1: Initial version, nothing special. -- 2: [Ensure consistent permissions](https://github.com/NixOS/nixpkgs/pull/422975) +- 1: Initial version, nothing special. (removed in 26.11) +- 2: [Ensure consistent permissions](https://github.com/NixOS/nixpkgs/pull/422975) (removed in 26.11) - 3: [Build a reproducible tarball](https://github.com/NixOS/nixpkgs/pull/469950) - 4: [Dump SQLite database to an SQL file](https://github.com/NixOS/nixpkgs/pull/522703) diff --git a/doc/release-notes/rl-2611.section.md b/doc/release-notes/rl-2611.section.md index 1df5e6b9a01c..5e674df1f8da 100644 --- a/doc/release-notes/rl-2611.section.md +++ b/doc/release-notes/rl-2611.section.md @@ -29,6 +29,13 @@ - `librest` providing 0.7 ABI was removed. `librest_1_0` providing 1.0 ABI was renamed to `librest` and `librest_1_0` was kept as an alias. +- `fetchPnpmDeps`' `fetcherVersion = 1` and `fetcherVersion = 2` have been + removed, as announced in the 26.05 release. Packages still using them now + throw an evaluation error and must migrate to `fetcherVersion = 3` (or later) + and regenerate their hashes. See the + [pnpm `fetcherVersion` section](#javascript-pnpm-fetcherVersion) of the manual + for details. + ## Other Notable Changes {#sec-nixpkgs-release-26.11-notable-changes} diff --git a/pkgs/build-support/node/fetch-pnpm-deps/default.nix b/pkgs/build-support/node/fetch-pnpm-deps/default.nix index 65d7063efac0..58e6370404ae 100644 --- a/pkgs/build-support/node/fetch-pnpm-deps/default.nix +++ b/pkgs/build-support/node/fetch-pnpm-deps/default.nix @@ -17,8 +17,6 @@ let pnpmLatest = pnpm; supportedFetcherVersions = [ - 1 # First version. Here to preserve backwards compatibility - 2 # Ensure consistent permissions. See https://github.com/NixOS/nixpkgs/pull/422975 3 # Build a reproducible tarball. See https://github.com/NixOS/nixpkgs/pull/469950 4 # Dump SQLite database to an SQL file. See https://github.com/NixOS/nixpkgs/pull/522703 ]; @@ -63,178 +61,167 @@ in fetcherVersion != null || throw "fetchPnpmDeps: `fetcherVersion` is not set, see https://nixos.org/manual/nixpkgs/stable/#javascript-pnpm-fetcherVersion."; + assert + !(fetcherVersion == 1 || fetcherVersion == 2) + || throw "fetchPnpmDeps: `fetcherVersion = ${toString fetcherVersion}` was removed in the 26.11 release. Please migrate `${pname}` to `fetcherVersion = 3` and regenerate the hash. See https://nixos.org/manual/nixpkgs/stable/#javascript-pnpm-fetcherVersion."; + assert builtins.elem fetcherVersion supportedFetcherVersions || throw "fetchPnpmDeps `fetcherVersion` is not set to a supported value (${lib.concatStringsSep ", " (map toString supportedFetcherVersions)}), see https://nixos.org/manual/nixpkgs/stable/#javascript-pnpm-fetcherVersion."; - lib.warnIf (fetcherVersion < 3) - "fetchPnpmDeps: `fetcherVersion = ${toString fetcherVersion}` is deprecated and scheduled for removal in the 26.11 release. Please migrate `${pname}` to `fetcherVersion = 3` and regenerate the hash. See https://nixos.org/manual/nixpkgs/stable/#javascript-pnpm-fetcherVersion." - - stdenvNoCC.mkDerivation + stdenvNoCC.mkDerivation ( + finalAttrs: ( - finalAttrs: - ( - args' - // { - name = "${pname}-pnpm-deps"; + args' + // { + name = "${pname}-pnpm-deps"; - nativeBuildInputs = [ - cacert - jq - moreutils - pnpm # from args - pnpm-fixup-state-db' - sqlite - writableTmpDirAsHomeHook - yq - zstd - ] - ++ args.nativeBuildInputs or [ ]; + nativeBuildInputs = [ + cacert + jq + moreutils + pnpm # from args + pnpm-fixup-state-db' + sqlite + writableTmpDirAsHomeHook + yq + zstd + ] + ++ args.nativeBuildInputs or [ ]; - impureEnvVars = - lib.fetchers.proxyImpureEnvVars ++ [ "NIX_NPM_REGISTRY" ] ++ args.impureEnvVars or [ ]; + impureEnvVars = + lib.fetchers.proxyImpureEnvVars ++ [ "NIX_NPM_REGISTRY" ] ++ args.impureEnvVars or [ ]; - installPhase = '' - runHook preInstall + installPhase = '' + runHook preInstall - versionAtLeast () { - local cur_version=$1 min_version=$2 - printf "%s\0%s" "$min_version" "$cur_version" | sort -zVC - } + versionAtLeast () { + local cur_version=$1 min_version=$2 + printf "%s\0%s" "$min_version" "$cur_version" | sort -zVC + } - lockfileVersion="$(yq -r .lockfileVersion pnpm-lock.yaml)" - if [[ ''${lockfileVersion:0:1} -gt ${lib.versions.major pnpm.version} ]]; then - echo "ERROR: lockfileVersion $lockfileVersion in pnpm-lock.yaml is too new for the provided pnpm version ${lib.versions.major pnpm.version}!" - exit 1 + lockfileVersion="$(yq -r .lockfileVersion pnpm-lock.yaml)" + if [[ ''${lockfileVersion:0:1} -gt ${lib.versions.major pnpm.version} ]]; then + echo "ERROR: lockfileVersion $lockfileVersion in pnpm-lock.yaml is too new for the provided pnpm version ${lib.versions.major pnpm.version}!" + exit 1 + fi + + # The pnpm store is bundled into a compressed tarball within $out, + # without distributing the uncompressed store files. + mkdir $out + storePath=$(mktemp -d) + + pushd "$HOME" + pnpmVersion=$(pnpm --version) + + if versionAtLeast "$pnpmVersion" "11"; then + # pnpm 11 uses a different mechanism to manage package manager versions + export pnpm_config_pm_on_fail=ignore + + # Some packages produce platform dependent outputs. We do not want to cache those in the global store + export pnpm_config_side_effects_cache=false + + export pnpm_config_update_notifier=false + else + pnpm config set manage-package-manager-versions false + pnpm config set side-effects-cache false + pnpm config set update-notifier false + fi + popd + + pnpm config set store-dir $storePath + + # Run any additional pnpm configuration commands that users provide. + ${prePnpmInstall} + + echo "Final pnpm config:" + pnpm config list + echo + + # pnpm is going to warn us about using --force + # --force allows us to fetch all dependencies including ones that aren't meant for our host platform + pnpm install \ + --force \ + --ignore-scripts \ + ${lib.escapeShellArgs filterFlags} \ + ${lib.escapeShellArgs pnpmInstallFlags} \ + --registry="$NIX_NPM_REGISTRY" \ + --frozen-lockfile + + # Record the fetcherVersion in the output for introspection. + echo ${toString fetcherVersion} > $out/.fetcher-version + + runHook postInstall + ''; + + fixupPhase = '' + runHook preFixup + + # Remove timestamp and sort the json files + rm -rf $storePath/{v3,v10,v11}/tmp + for f in $(find $storePath -name "*.json"); do + jq --sort-keys "del(.. | .checkedAt?)" $f | sponge $f + done + + if [ -f "$storePath/v11/index.db" ]; then + pnpm-fixup-state-db "$storePath/v11"; + # Dump the SQLite database to a SQL text file for reproducibility. + # SQLite's binary format is non-deterministic (version-valid-for number, etc), + # so we store the logical contents as SQL statements and reconstruct during build. + if [[ ${toString fetcherVersion} -ge 4 ]]; then + sqlite3 "$storePath/v11/index.db" .dump > "$storePath/v11/index.db.sql" + rm "$storePath/v11/index.db" fi + fi - # For fetcherVersion < 3, the pnpm store files are placed directly into $out. - # For fetcherVersion >= 3, it is bundled into a compressed tarball within $out, - # without distributing the uncompressed store files. - if [[ ${toString fetcherVersion} -ge 3 ]]; then - mkdir $out - storePath=$(mktemp -d) - else - storePath=$out - fi + # This folder contains symlinks to /build/source which we don't need + # since https://github.com/pnpm/pnpm/releases/tag/v10.27.0 + rm -rf $storePath/{v3,v10,v11}/projects - pushd "$HOME" - pnpmVersion=$(pnpm --version) + # Ensure consistent permissions + # NOTE: For reasons not yet fully understood, pnpm might create files with + # inconsistent permissions, for example inside the ubuntu-24.04 + # github actions runner. + # To ensure stable derivations, we need to set permissions + # consistently, namely: + # * All files with `-exec` suffix have 555. + # * All other files have 444. + # * All folders have 555. + # See https://github.com/NixOS/nixpkgs/pull/350063 + # See https://github.com/NixOS/nixpkgs/issues/422889 + find $storePath -type f -name "*-exec" -print0 | xargs --no-run-if-empty -0 chmod 555 + find $storePath -type f -not -name "*-exec" -print0 | xargs --no-run-if-empty -0 chmod 444 + find $storePath -type d -print0 | xargs --no-run-if-empty -0 chmod 555 - if versionAtLeast "$pnpmVersion" "11"; then - # pnpm 11 uses a different mechanism to manage package manager versions - export pnpm_config_pm_on_fail=ignore + ( + cd $storePath - # Some packages produce platform dependent outputs. We do not want to cache those in the global store - export pnpm_config_side_effects_cache=false + # Build a reproducible tarball, per instructions at https://reproducible-builds.org/docs/archives/ + tar --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 --numeric-owner \ + --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ + --zstd -cf $out/pnpm-store.tar.zst . + ) - export pnpm_config_update_notifier=false - else - pnpm config set manage-package-manager-versions false - pnpm config set side-effects-cache false - pnpm config set update-notifier false - fi - popd + runHook postFixup + ''; - pnpm config set store-dir $storePath - - # Run any additional pnpm configuration commands that users provide. - ${prePnpmInstall} - - echo "Final pnpm config:" - pnpm config list - echo - - # pnpm is going to warn us about using --force - # --force allows us to fetch all dependencies including ones that aren't meant for our host platform - pnpm install \ - --force \ - --ignore-scripts \ - ${lib.escapeShellArgs filterFlags} \ - ${lib.escapeShellArgs pnpmInstallFlags} \ - --registry="$NIX_NPM_REGISTRY" \ - --frozen-lockfile - - # Store newer fetcherVersion in case pnpmConfigHook also needs it - if [[ ${toString fetcherVersion} -gt 1 ]]; then - echo ${toString fetcherVersion} > $out/.fetcher-version - fi - - runHook postInstall - ''; - - fixupPhase = '' - runHook preFixup - - # Remove timestamp and sort the json files - rm -rf $storePath/{v3,v10,v11}/tmp - for f in $(find $storePath -name "*.json"); do - jq --sort-keys "del(.. | .checkedAt?)" $f | sponge $f - done - - if [ -f "$storePath/v11/index.db" ]; then - pnpm-fixup-state-db "$storePath/v11"; - # Dump the SQLite database to a SQL text file for reproducibility. - # SQLite's binary format is non-deterministic (version-valid-for number, etc), - # so we store the logical contents as SQL statements and reconstruct during build. - if [[ ${toString fetcherVersion} -ge 4 ]]; then - sqlite3 "$storePath/v11/index.db" .dump > "$storePath/v11/index.db.sql" - rm "$storePath/v11/index.db" - fi - fi - - # This folder contains symlinks to /build/source which we don't need - # since https://github.com/pnpm/pnpm/releases/tag/v10.27.0 - rm -rf $storePath/{v3,v10,v11}/projects - - # Ensure consistent permissions - # NOTE: For reasons not yet fully understood, pnpm might create files with - # inconsistent permissions, for example inside the ubuntu-24.04 - # github actions runner. - # To ensure stable derivations, we need to set permissions - # consistently, namely: - # * All files with `-exec` suffix have 555. - # * All other files have 444. - # * All folders have 555. - # See https://github.com/NixOS/nixpkgs/pull/350063 - # See https://github.com/NixOS/nixpkgs/issues/422889 - if [[ ${toString fetcherVersion} -ge 2 ]]; then - find $storePath -type f -name "*-exec" -print0 | xargs --no-run-if-empty -0 chmod 555 - find $storePath -type f -not -name "*-exec" -print0 | xargs --no-run-if-empty -0 chmod 444 - find $storePath -type d -print0 | xargs --no-run-if-empty -0 chmod 555 - fi - - if [[ ${toString fetcherVersion} -ge 3 ]]; then - ( - cd $storePath - - # Build a reproducible tarball, per instructions at https://reproducible-builds.org/docs/archives/ - tar --sort=name \ - --mtime="@$SOURCE_DATE_EPOCH" \ - --owner=0 --group=0 --numeric-owner \ - --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ - --zstd -cf $out/pnpm-store.tar.zst . - ) - fi - - runHook postFixup - ''; - - passthru = args.passthru or { } // { - inherit fetcherVersion; - serve = callPackage ./serve.nix { - inherit pnpm; # from args - pnpmDeps = finalAttrs.finalPackage; - }; + passthru = args.passthru or { } // { + inherit fetcherVersion; + serve = callPackage ./serve.nix { + inherit pnpm; # from args + pnpmDeps = finalAttrs.finalPackage; }; + }; - dontConfigure = true; - dontBuild = true; - outputHashMode = "recursive"; - } - // hash' - ) + dontConfigure = true; + dontBuild = true; + outputHashMode = "recursive"; + } + // hash' ) + ) ); pnpmConfigHook = makeSetupHook { diff --git a/pkgs/build-support/node/fetch-pnpm-deps/pnpm-config-hook.sh b/pkgs/build-support/node/fetch-pnpm-deps/pnpm-config-hook.sh index 30c8bd444c24..5cafdd6d1d8b 100644 --- a/pkgs/build-support/node/fetch-pnpm-deps/pnpm-config-hook.sh +++ b/pkgs/build-support/node/fetch-pnpm-deps/pnpm-config-hook.sh @@ -40,7 +40,7 @@ pnpmConfigHook() { echo "Found 'pnpm' with version '$pnpmVersion'" - fetcherVersion=$(cat "${pnpmDeps}/.fetcher-version" || echo 1) + fetcherVersion=$(cat "${pnpmDeps}/.fetcher-version") echo "Using fetcherVersion: $fetcherVersion" @@ -52,11 +52,7 @@ pnpmConfigHook() { export npm_config_platform="@npmPlatform@" export pnpm_config_platform="@npmPlatform@" - if [[ $fetcherVersion -ge 3 ]]; then - tar --zstd -xf "$pnpmDeps/pnpm-store.tar.zst" -C "$STORE_PATH" - else - cp -Tr "$pnpmDeps" "$STORE_PATH" - fi + tar --zstd -xf "$pnpmDeps/pnpm-store.tar.zst" -C "$STORE_PATH" chmod -R +w "$STORE_PATH" diff --git a/pkgs/build-support/node/fetch-pnpm-deps/serve.nix b/pkgs/build-support/node/fetch-pnpm-deps/serve.nix index 46288e974660..2c3292180d8f 100644 --- a/pkgs/build-support/node/fetch-pnpm-deps/serve.nix +++ b/pkgs/build-support/node/fetch-pnpm-deps/serve.nix @@ -17,8 +17,6 @@ writeShellApplication { text = '' storePath=$(mktemp -d) - fetcherVersion=$(cat "${pnpmDeps}/.fetcher-version" || echo 1) - clean() { echo "Cleaning up temporary store at '$storePath'..." @@ -27,11 +25,7 @@ writeShellApplication { echo "Copying pnpm store '${pnpmDeps}' to temporary store..." - if [[ $fetcherVersion -ge 3 ]]; then - tar --zstd -xf "${pnpmDeps}/pnpm-store.tar.zst" -C "$storePath" - else - cp -Tr "${pnpmDeps}" "$storePath" - fi + tar --zstd -xf "${pnpmDeps}/pnpm-store.tar.zst" -C "$storePath" chmod -R +w "$storePath"