From 48e760239859a230a3d8e50f203acc7db7575bd8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 6 Mar 2026 09:35:43 -0800 Subject: [PATCH 1/6] fetchtorrent: use a random name for downloadedDirectory Generate a random name for `downloadDirectory` in case the torrent contains a directory called `downloadDirectory`. This is unlikely to happen in practice, but we might as well play it safe. --- pkgs/build-support/fetchtorrent/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index 5eb6aa5f7241..4dc41b0edcd8 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -116,8 +116,8 @@ runCommand name if (backend == "transmission") then '' export HOME=$TMP - export downloadedDirectory=$out/downloadedDirectory - mkdir -p $downloadedDirectory + mkdir -p $out + export downloadedDirectory=$(mktemp -d $out/downloadedDirectory.XXXXXXXXXX) mkdir -p $HOME/.config/transmission cp ${jsonConfig} $HOME/.config/transmission/settings.json port="$(shuf -n 1 -i 49152-65535)" @@ -139,8 +139,8 @@ runCommand name export HOME=$TMP '' + lib.optionalString flatten' '' - downloadedDirectory=$out/downloadedDirectory - mkdir -p $downloadedDirectory + mkdir -p $out + downloadedDirectory=$(mktemp -d $out/downloadedDirectory.XXXXXXXXXX) '' + lib.optionalString (!flatten') '' downloadedDirectory=$out From 2e3d501f48e2a882f8f6d44868ecdb1898075d07 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 6 Mar 2026 09:38:11 -0800 Subject: [PATCH 2/6] fetchtorrent: handle single-file downloads Previously, we'd attempt to flatten all downloads with: mv -v $downloadedDir/*/* $out However, this doesn't work when we download a single file to `$downloadedDir/some-file-name`. This patch handles this case as follows: 1. If no files were downloaded, we exit with an error. 2. If a single directory was downloaded, we flatten it. 3. If a single file was downloaded we move it to $out. 4. If multiple files/directories were downloaded directly to `$downloadedDir`, we move all these files into $out directly. I haven't encountered this last case, but it doesn't add any complexity to support it. --- pkgs/build-support/fetchtorrent/default.nix | 76 +++++++++++---------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index 4dc41b0edcd8..36ace6b9e2a8 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -31,15 +31,45 @@ let # Default to flattening if no flatten argument was specified. flatten' = if flatten == null then true else flatten; - transmissionFinishScript = writeShellScript "fetch-bittorrent-done.sh" '' + setupScript = '' + export HOME=$TMP + mkdir -p $out + downloadedDirectory=${ + if flatten' then "$(mktemp -d $out/downloadedDirectory.XXXXXXXXXX)" else "$out" + } + export downloadedDirectory # See https://www.shellcheck.net/wiki/SC2155 + port="$(shuf -n 1 -i 49152-65535)" + ''; + + flattenScript = '' + ( + shopt -s dotglob nullglob + downloadedFiles=($downloadedDirectory/*) + if [[ ''${#downloadedFiles[@]} -eq 0 ]]; then + echo "Failed to download any files." + exit 1 + elif [[ ''${#downloadedFiles[@]} -eq 1 ]] && [[ -d "$downloadedFiles" ]]; then + # Flatten the directory, so that only the torrent contents are in $out, + # not the folder name + mv -v "$downloadedFiles"/* $out + else + # Either we downloaded a single (regular) file, or we downloaded + # multiple files/directories. We can't flatten, so we move them to + # $out. + mv -v "''${downloadedFiles[@]}" $out + fi + rm -v -rf $downloadedDirectory + ) + ''; + + finishScript = '' ${postUnpack} - # Flatten the directory, so that only the torrent contents are in $out, not - # the folder name - shopt -s dotglob - mv -v $downloadedDirectory/*/* $out - rm -v -rf $downloadedDirectory - unset downloadedDirectory + ${lib.optionalString flatten' flattenScript} ${postFetch} + ''; + + transmissionFinishScript = writeShellScript "fetch-bittorrent-done.sh" '' + ${finishScript} kill $PPID ''; jsonConfig = (formats.json { }).generate "jsonConfig" config; @@ -115,12 +145,9 @@ runCommand name ( if (backend == "transmission") then '' - export HOME=$TMP - mkdir -p $out - export downloadedDirectory=$(mktemp -d $out/downloadedDirectory.XXXXXXXXXX) + ${setupScript} mkdir -p $HOME/.config/transmission cp ${jsonConfig} $HOME/.config/transmission/settings.json - port="$(shuf -n 1 -i 49152-65535)" function handleChild { # This detects failures and logs the contents of the transmission fetch find $out @@ -136,18 +163,7 @@ runCommand name '' else '' - export HOME=$TMP - '' - + lib.optionalString flatten' '' - mkdir -p $out - downloadedDirectory=$(mktemp -d $out/downloadedDirectory.XXXXXXXXXX) - '' - + lib.optionalString (!flatten') '' - downloadedDirectory=$out - '' - + '' - port="$(shuf -n 1 -i 49152-65535)" - + ${setupScript} rqbit \ --disable-dht-persistence \ --http-api-listen-addr "127.0.0.1:$port" \ @@ -155,18 +171,6 @@ runCommand name -o "$downloadedDirectory" \ --exit-on-finish \ "$url" - - ${postUnpack} - '' - + lib.optionalString flatten' '' - # Flatten the directory, so that only the torrent contents are in $out, - # not the folder name - shopt -s dotglob - mv -v $downloadedDirectory/*/* $out - rm -v -rf $downloadedDirectory - unset downloadedDirectory - '' - + '' - ${postFetch} + ${finishScript} '' ) From a59ff0d1231d6773e5a1ce377b722e888f29ac68 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 6 Mar 2026 11:10:31 -0800 Subject: [PATCH 3/6] fetchtorrent: actually emit flatten warnings Previously, they were let-bound to a `warnings` variable, but never used. --- pkgs/build-support/fetchtorrent/default.nix | 75 ++++++++++----------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix index 36ace6b9e2a8..ae411881c7c9 100644 --- a/pkgs/build-support/fetchtorrent/default.nix +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -73,45 +73,6 @@ let kill $PPID ''; jsonConfig = (formats.json { }).generate "jsonConfig" config; - - # https://github.com/NixOS/nixpkgs/issues/432001 - # - # For a while, the transmission backend would put the downloaded torrent in - # the output directory, but whether the rqbit backend would put the output in - # the output directory or a subdirectory depended on the version of rqbit. - # We want to standardise on a single behaviour, but give users of - # fetchtorrent with the rqbit backend some warning that the behaviour might - # be unexpected, particularly since we can't know what behaviour users might - # be expecting at this point, and they probably wouldn't notice a change - # straight away because the results are fixed-output derivations. - # - # This warning was introduced for 25.11, so we can remove handling of the - # `flatten` argument once that release is no longer supported. - warnings = - if backend == "rqbit" && flatten == null then - [ - '' - `fetchtorrent` with the rqbit backend may or may not have the - downloaded files stored in a subdirectory of the output directory. - Verify which behaviour you need, and set the `flatten` argument to - `fetchtorrent` accordingly. - - The `flatten = false` behaviour will still produce a warning, as this - behaviour is deprecated. It is only available with the "rqbit" backend - to provide temporary support for users who are relying on the - previous incorrect behaviour. For a warning-free evaluation, use - `flatten = true`. - '' - ] - else if flatten == false then - [ - '' - `fetchtorrent` with `flatten = false` is deprecated and will be - removed in a future release. - '' - ] - else - [ ]; in assert lib.assertMsg (config != { } -> backend == "transmission") '' json config for configuring fetchtorrent only works with the transmission backend @@ -129,7 +90,41 @@ runCommand name if (backend == "transmission") then [ transmission_4 ] else if (backend == "rqbit") then - [ rqbit ] + lib.warnIf (flatten != true) ( + # https://github.com/NixOS/nixpkgs/issues/432001 + # + # For a while, the transmission backend would put the downloaded torrent in + # the output directory, but whether the rqbit backend would put the output in + # the output directory or a subdirectory depended on the version of rqbit. + # We want to standardise on a single behaviour, but give users of + # fetchtorrent with the rqbit backend some warning that the behaviour might + # be unexpected, particularly since we can't know what behaviour users might + # be expecting at this point, and they probably wouldn't notice a change + # straight away because the results are fixed-output derivations. + # + # This warning was introduced for 25.11, so we can remove handling of the + # `flatten` argument once that release is no longer supported. + if flatten == null then + '' + `fetchtorrent` with the rqbit backend may or may not have the + downloaded files stored in a subdirectory of the output directory. + Verify which behaviour you need, and set the `flatten` argument to + `fetchtorrent` accordingly. + + The `flatten = false` behaviour will still produce a warning, as this + behaviour is deprecated. It is only available with the "rqbit" backend + to provide temporary support for users who are relying on the + previous incorrect behaviour. For a warning-free evaluation, use + `flatten = true`. + '' + else if flatten == false then + '' + `fetchtorrent` with `flatten = false` is deprecated and will be + removed in a future release. + '' + else + throw "invalid value for flatten: must be true, false, or null" + ) [ rqbit ] else throw "rqbit or transmission are the only available backends for fetchtorrent" ); From b20865b73214f86db1fbec25ed7ccb4835f69183 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 6 Mar 2026 18:15:26 -0800 Subject: [PATCH 4/6] fetchtorrent: disable tests that now warn about future deprecation --- pkgs/build-support/fetchtorrent/tests.nix | 56 ++++++++++++++--------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix index 69d47d589998..bfefdb942ddf 100644 --- a/pkgs/build-support/fetchtorrent/tests.nix +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -94,16 +94,22 @@ builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash backend = "transmission"; inherit (flattened) postFetch; }; - http-link-rqbit = { - inherit (http) url; - backend = "rqbit"; - inherit (flattened) postFetch; - }; - magnet-link-rqbit = { - inherit (magnet) url; - backend = "rqbit"; - inherit (flattened) postFetch; - }; + # + # Disabled because these warn that flatten hasn't been explicitly + # set to true, and warnings are treated as failures in tests. + # + # Re-enable these tests when flatten defaults to true. + # + #http-link-rqbit = { + # inherit (http) url; + # backend = "rqbit"; + # inherit (flattened) postFetch; + #}; + #magnet-link-rqbit = { + # inherit (magnet) url; + # backend = "rqbit"; + # inherit (flattened) postFetch; + #}; http-link-rqbit-flattened = { inherit (http) url; backend = "rqbit"; @@ -116,16 +122,22 @@ builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash flatten = true; inherit (flattened) postFetch; }; - http-link-rqbit-unflattened = { - inherit (http) url; - backend = "rqbit"; - flatten = false; - inherit (unflattened) postFetch; - }; - magnet-link-rqbit-unflattened = { - inherit (magnet) url; - backend = "rqbit"; - flatten = false; - inherit (unflattened) postFetch; - }; + # + # Disabled because these warn that `flatten = false` is deprecated + # and will be removed. + # + # Remove these when support for `flatten = false` is completely removed. + # + #http-link-rqbit-unflattened = { + # inherit (http) url; + # backend = "rqbit"; + # flatten = false; + # inherit (unflattened) postFetch; + #}; + #magnet-link-rqbit-unflattened = { + # inherit (magnet) url; + # backend = "rqbit"; + # flatten = false; + # inherit (unflattened) postFetch; + #}; } From d0a09641e4309f2f765a69ac2690b5855a53c4c9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 7 Mar 2026 16:17:15 -0800 Subject: [PATCH 5/6] fetchtorrent: actually fail tests when they fail This is only an issue in these tests because we're deleting the test output and relying on the derivation to exit with a non-zero exit code. Usually, if fetching a torrent fails, the output hash will differ (modulo issues with TOFU). --- pkgs/build-support/fetchtorrent/tests.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix index bfefdb942ddf..c4f61aa30ad9 100644 --- a/pkgs/build-support/fetchtorrent/tests.nix +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -47,7 +47,7 @@ let pushd "$out" && sha512sum --check --strict ${./test-hashes.sha512sum} && sed 's/.* //' ${./test-hashes.sha512sum} | xargs rm --verbose && - popd + popd || exit 1 ''; unflattened.postFetch = '' pushd "$out" && @@ -56,7 +56,7 @@ let sed 's/.* //' ${./test-hashes.sha512sum} | xargs rm --verbose && popd && rm --dir --verbose Sintel && - popd + popd || exit 1 ''; fetchtorrentWithHash = From b077f0d1170ea5eb76161f1da405a20c9ab556a5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 7 Mar 2026 16:21:29 -0800 Subject: [PATCH 6/6] fetchtorrent: test single-file torrents --- .editorconfig | 2 +- .gitattributes | 3 +++ .../fetchtorrent/test-single-file.torrent | 1 + pkgs/build-support/fetchtorrent/tests.nix | 18 +++++++++++++++++- 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 pkgs/build-support/fetchtorrent/test-single-file.torrent diff --git a/.editorconfig b/.editorconfig index 3a2159d33734..873d7efb1372 100644 --- a/.editorconfig +++ b/.editorconfig @@ -79,7 +79,7 @@ indent_size = unset trim_trailing_whitespace = true # binaries -[*.nib] +[*.{nib,torrent}] end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset diff --git a/.gitattributes b/.gitattributes index d67974e66b69..2c113f5fd5dc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -62,3 +62,6 @@ ci/OWNERS linguist-language=CODEOWNERS # patching CRLF line endings from an upstream source package. *.diff !text !eol *.patch !text !eol + +# Torrent files are binary files and should not be re-encoded. +*.torrent !text !eol diff --git a/pkgs/build-support/fetchtorrent/test-single-file.torrent b/pkgs/build-support/fetchtorrent/test-single-file.torrent new file mode 100644 index 000000000000..0d8ecae342d7 --- /dev/null +++ b/pkgs/build-support/fetchtorrent/test-single-file.torrent @@ -0,0 +1 @@ +d10:created by13:mktorrent 1.113:creation datei1772926249e4:infod6:lengthi1097e4:name7:COPYING12:piece lengthi262144e6:pieces20:€åÏÊ\c™èçæÉ\Q3Ní-e8:url-list96:https://raw.githubusercontent.com/NixOS/nixpkgs/3d82431ec4b51262989559b821b83abe4a9f6dbd/COPYINGe \ No newline at end of file diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix index c4f61aa30ad9..ef14999254ff 100644 --- a/pkgs/build-support/fetchtorrent/tests.nix +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -75,7 +75,7 @@ let ); in # Seems almost but not quite worth using lib.mapCartesianProduct... -builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash v) { +(builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash v) { http-link = { inherit (http) url; inherit (flattened) postFetch; @@ -140,4 +140,20 @@ builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash # flatten = false; # inherit (unflattened) postFetch; #}; +}) +// { + # Make sure we can download and "flatten" single-file torrents. + # We only test with transmission because we use a web-seed and rqbit + # doesn't support web-seeds. + single-file = fetchtorrent { + hash = "sha256-u5c/ZN5V79Jg1aN6spbui4OdhExsXjofk1JpvUdW7Ro="; + backend = "transmission"; + flatten = true; + url = "${./test-single-file.torrent}"; + meta = { + hydraPlatforms = [ ]; + license = lib.licenses.mit; + description = "The license file for the Nixpkgs repository as of 7 March 2026"; + }; + }; }