From 96d5ebc8c2e94a4ad2685ecca393e98ad6d9dca4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Sun, 16 Nov 2025 21:14:20 +1300 Subject: [PATCH 1/5] lib.sources.sourceByGlobs: init function Adds a source filtering function inspired by [doublestar](https://github.com/bmatcuk/doublestar). This has been in used in a few private repositories since the last ~6 months with success. - Testing This was originally tested with the nix-unit testsuite: ``` let inherit (import ./internal.nix) mkSourceFilter mkMatcher; in { mkMatcher = { empty = { testMatch = { expr = mkMatcher "" "" "regular"; expected = true; }; testNoMatch = { expr = mkMatcher "" "foo" "regular"; expected = false; }; }; simple = { testMatch = { expr = mkMatcher "foo" "foo" "regular"; expected = true; }; testNoMatch = { expr = mkMatcher "foo" "bar" "regular"; expected = false; }; }; singleStar = { testMatch = { expr = mkMatcher "*.js" "foo.js" "regular"; expected = true; }; testNoMatch = { expr = mkMatcher "*.js" "foo.py" "regular"; expected = false; }; }; doubleStar = { testMatch = { expr = mkMatcher "foo/**/bar" "foo/baz/bar" "regular"; expected = true; }; testNoMatch = { expr = mkMatcher "foo/**/bar" "foo/bar/baz" "regular"; expected = false; }; testMultiMatch = { expr = mkMatcher "foo/**/bar" "foo/baz/xyz/bar" "regular"; expected = true; }; testMultiMatchDoubleGlob = { expr = mkMatcher "foo/**/**/bar" "foo/baz/xyz/bar" "regular"; expected = true; }; testInfixMatch = { expr = mkMatcher "foo/**/qux/**/bar" "foo/baz/qux/baz/bar" "regular"; expected = true; }; testInfixNoMatch = { expr = mkMatcher "foo/**/xyz/**/bar" "foo/baz/qux/baz/bar" "regular"; expected = false; }; # Technically a partial match testInfixDirMatch = { expr = mkMatcher "foo/**/xyz/**/bar" "foo/baz/qux/baz/bar" "directory"; expected = true; }; }; }; mkSourceFilter = { testSourceFilter = { expr = mkSourceFilter ./fixtures [ "bar/*.js" ] "bar/bar.js" "regular"; expected = true; }; }; } ``` but it was dropped in this nixpkgs contribution as the structure of nixpkgs lib testing is too primitive to incorp this without more extensive refactoring than I'd like at the momment. - Performance It's hard to benchmark this against anything else meaningful except [globsset](https://github.com/pdtpartners/globset), which has a very similar API. `sourceByGlobs` avoids performance pitfalls by: - Using `builtins.filterSource` This is more performant than the fileset API. The downside compared to the fileset API is that any directory which matches the filter will be added to the build, even if it's empty. - Match paths component by component By splitting each pattern into a token per / separator. This is much faster in Nix than the doublestar algorithm. - Globset source ```json { "cpuTime": 0.8585879802703857, "envs": { "bytes": 148252864, "elements": 11899843, "number": 6631765 }, "gc": { "heapSize": 402915328, "totalBytes": 671288560 }, "list": { "bytes": 3358664, "concats": 28658, "elements": 419833 }, "nrAvoided": 11562713, "nrFunctionCalls": 4816963, "nrLookups": 4316209, "nrOpUpdateValuesCopied": 5686407, "nrOpUpdates": 464060, "nrPrimOpCalls": 2966970, "nrThunks": 7796186, "sets": { "bytes": 196404672, "elements": 10837802, "number": 1437490 }, "sizes": { "Attr": 16, "Bindings": 16, "Env": 8, "Value": 24 }, "symbols": { "bytes": 340652, "number": 32026 }, "values": { "bytes": 207367440, "number": 8640310 } } ``` - Glob-filter source ```json { "cpuTime": 0.3904629945755005, "envs": { "bytes": 13263440, "elements": 1005877, "number": 652053 }, "gc": { "heapSize": 402915328, "totalBytes": 146914896 }, "list": { "bytes": 3032168, "concats": 5899, "elements": 379021 }, "nrAvoided": 1666598, "nrFunctionCalls": 484399, "nrLookups": 112698, "nrOpUpdateValuesCopied": 3432135, "nrOpUpdates": 13426, "nrPrimOpCalls": 1041954, "nrThunks": 1205792, "sets": { "bytes": 64304800, "elements": 3978167, "number": 40883 }, "sizes": { "Attr": 16, "Bindings": 16, "Env": 8, "Value": 24 }, "symbols": { "bytes": 285306, "number": 28864 }, "values": { "bytes": 42963240, "number": 1790135 } } ``` (cherry picked from commit 59d55cbaa3f3e323167a0ba8ee262b7218a709e2) --- lib/sources.nix | 109 +++++++++++++++++++++++++++++++++++++++++++ lib/tests/sources.sh | 12 +++++ 2 files changed, 121 insertions(+) diff --git a/lib/sources.nix b/lib/sources.nix index 43bc2ab2dbe3..81c5305842a1 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -7,12 +7,19 @@ let match split storeDir + escapeRegex + removePrefix ; inherit (lib) boolToString filter isString readFile + concatStrings + length + elemAt + isList + any ; inherit (lib.filesystem) pathIsRegularFile @@ -513,6 +520,107 @@ let else throw "repoRevToName: invalid kind"; + /** + Filter sources by a list of double star glob patterns. + + # Inputs + + `src` + + : 1\. Function argument + + `patterns` + + : 2\. Function argument + + # Examples + :::{.example} + ## `sourceByGlobs` usage example + + - Include all .py files recursively + ```nix + src = sourceByGlobs ./my-subproject ["**\/*.py" ] + ``` + + - Include all .py files in root directory only + ```nix + src = sourceByGlobs ./my-subproject ["*.py" ] + ``` + + ::: + */ + sourceByGlobs = + let + splitPath = path: filter isString (split "\/" path); + # Make component regex + mkRe = + s: + if s == "**" then + ".*" # Has special handling below + else + concatStrings (map (tok: if isList tok then "[^\/]*" else escapeRegex tok) (split "\\*+" s)); + + # Make a source filter function from pattern + mkMatcher = + pat: + let + globs = map mkRe (splitPath pat); + glen = length globs; + in + path: type: + let + path' = splitPath path; + plen = length path'; + + recurse = + gi: pi: + let + g = elemAt globs gi; + p = elemAt path' pi; + m = match g p != null; + in + if pi >= plen then # Reached end of path + gi >= glen || (type == "directory" || type == "symlink") # Only allow partial matches for directories + else if gi >= glen then # Reached end of globs + false + else if g == ".*" then # Special handling for ** + ( + # Lookahead for next glob match + if (gi + 1) == glen then + true + else if (match (elemAt globs (gi + 1)) p != null) then + recurse (gi + 1) pi + else if m then + recurse gi (pi + 1) + else + false + ) + else if m then + recurse (gi + 1) (pi + 1) + else + false; + + in + recurse 0 0; + + mkSourceFilter = + root: patterns: + let + root' = "${toString root}/"; + matchers = map mkMatcher patterns; + in + name: type: + let + name' = removePrefix root' name; + in + any (m: m name' type) matchers; + + in + src: patterns: + lib.cleanSourceWith { + filter = mkSourceFilter src patterns; + inherit src; + }; in { inherit @@ -532,6 +640,7 @@ in sourceByRegex sourceFilesBySuffices + sourceByGlobs trace ; diff --git a/lib/tests/sources.sh b/lib/tests/sources.sh index 079c7eea5657..08315a303bc2 100755 --- a/lib/tests/sources.sh +++ b/lib/tests/sources.sh @@ -70,4 +70,16 @@ dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with im EOF ) || die "cleanSourceWith + cleanSource" + +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import ; "${ + sources.sourceByGlobs '"$work"' [ "*.md" "**/*.o" ] +}")' | crudeUnquoteJSON)" +(cd "$dir"; find) | sort -f | diff -U10 - <(cat <&2 tests ok From 8b870d76666838edc7e0b062f5bc3a5d5c4816cc Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 22 May 2026 14:14:40 +0200 Subject: [PATCH 2/5] lib.sources.sourceByGlobs: address review comments (cherry picked from commit 01bdc186ae9baf635bdfb732e3d81ac41da02ca4) --- lib/sources.nix | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/sources.nix b/lib/sources.nix index 81c5305842a1..43a374c16e73 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -521,44 +521,50 @@ let throw "repoRevToName: invalid kind"; /** - Filter sources by a list of double star glob patterns. + Filter a source tree by a list of doublestar-style glob patterns, + returning a source that only contains paths matching at least one + pattern. `*` matches a single path component, and `**` matches any + number of components. # Inputs `src` - : 1\. Function argument + : The source tree to filter. `patterns` - : 2\. Function argument + : List of glob patterns to include, e.g. `[ "*.py" "src/**" ]`. + A leading `**` (e.g. `**\/*.py` for all `.py` files at any depth) + is also supported; the `\` here is just a Nix string escape used + to avoid closing this comment. # Examples :::{.example} ## `sourceByGlobs` usage example - - Include all .py files recursively + - Include everything under a subdirectory ```nix - src = sourceByGlobs ./my-subproject ["**\/*.py" ] + src = sourceByGlobs ./. [ "src/**" "tests/**" ] ``` - Include all .py files in root directory only ```nix - src = sourceByGlobs ./my-subproject ["*.py" ] + src = sourceByGlobs ./. [ "*.py" ] ``` ::: */ sourceByGlobs = let - splitPath = path: filter isString (split "\/" path); + splitPath = path: filter isString (split "/" path); # Make component regex mkRe = s: if s == "**" then ".*" # Has special handling below else - concatStrings (map (tok: if isList tok then "[^\/]*" else escapeRegex tok) (split "\\*+" s)); + concatStrings (map (tok: if isList tok then "[^/]*" else escapeRegex tok) (split "\\*+" s)); # Make a source filter function from pattern mkMatcher = From 9dc23bf891b65d49130357aace01173b4e09e375 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 22 May 2026 14:42:16 +0200 Subject: [PATCH 3/5] pkgs.formats.javaProperties: use lib.sources.sourceByGlobs Uses the cleaner expression (cherry picked from commit edb4b053f526fa2cd42bdf459a298de622ce2f66) --- pkgs/pkgs-lib/formats/java-properties/test/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/pkgs-lib/formats/java-properties/test/default.nix b/pkgs/pkgs-lib/formats/java-properties/test/default.nix index 2ea787a90341..68cd5cfa1bcf 100644 --- a/pkgs/pkgs-lib/formats/java-properties/test/default.nix +++ b/pkgs/pkgs-lib/formats/java-properties/test/default.nix @@ -72,8 +72,8 @@ stdenv.mkDerivation { ) ); - src = lib.sourceByRegex ./. [ - ".*\\.java" + src = lib.sources.sourceByGlobs ./. [ + "**/*.java" ]; # On Linux, this can be C.UTF-8, but darwin + zulu requires en_US.UTF-8 LANG = "en_US.UTF-8"; From 576870d97cc2081b988a12ca46c4f857dbe7a8bc Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 22 May 2026 15:25:46 +0200 Subject: [PATCH 4/5] vimPluginsUpdater: use lib.sources.sourceByGlobs Filter PYTHONPATH source to .py files only, dropping the 6 sibling .nix files from the closure. (cherry picked from commit 1b476677762937fc0364a4f37bb9e31aebb61220) --- pkgs/applications/editors/vim/plugins/utils/updater.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/editors/vim/plugins/utils/updater.nix b/pkgs/applications/editors/vim/plugins/utils/updater.nix index 2ed61a25c83a..0c77e7ce745c 100644 --- a/pkgs/applications/editors/vim/plugins/utils/updater.nix +++ b/pkgs/applications/editors/vim/plugins/utils/updater.nix @@ -43,7 +43,7 @@ buildPythonApplication { neovim-unwrapped nurl ] - }" --prefix PYTHONPATH : "${./.}" ) + }" --prefix PYTHONPATH : "${lib.sources.sourceByGlobs ./. [ "**/*.py" ]}" ) wrapPythonPrograms ''; From ba1f4720f0869a51a2ff2f8632dd3efa167470d4 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 22 May 2026 15:25:46 +0200 Subject: [PATCH 5/5] purescript.tests.minimal-module: use lib.sources.sourceByGlobs Filter src to the .purs and .js files actually compiled, dropping default.nix from the closure. (cherry picked from commit df1b67fdfd63eb27e7b135033b9228bdf4ea8826) --- .../purescript/purescript/test-minimal-module/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkgs/development/compilers/purescript/purescript/test-minimal-module/default.nix b/pkgs/development/compilers/purescript/purescript/test-minimal-module/default.nix index a8503e1b9aae..2a0010b061a5 100644 --- a/pkgs/development/compilers/purescript/purescript/test-minimal-module/default.nix +++ b/pkgs/development/compilers/purescript/purescript/test-minimal-module/default.nix @@ -1,11 +1,17 @@ { + lib, runCommand, purescript, nodejs, }: runCommand "purescript-test-minimal-module" { } '' - ${purescript}/bin/purs compile -o ./output ${./.}/Main.purs + ${purescript}/bin/purs compile -o ./output ${ + lib.sources.sourceByGlobs ./. [ + "*.purs" + "*.js" + ] + }/Main.purs echo 'import {main} from "./output/Main/index.js"; main()' > node.mjs