Merge pull request #185612 from obsidiansystems/buildRustCrate-21.05-no-choke-on-dep

[Backport release 21.05] buildRustCrate: don't try to set CARGO_FEATURE_ variables for dep: fe…

pkgs/build-support/rust/build-rust-crate/build-crate.nix

@@ -1,4 +1,8 @@
-{ lib, stdenv, mkRustcDepArgs, mkRustcFeatureArgs, rust }:
+{ lib, stdenv
+, mkRustcDepArgs, mkRustcFeatureArgs, needUnstableCLI
+, rust
+}:
+
 { crateName,
   dependencies,
   crateFeatures, crateRenames, libName, release, libPath,
@@ -18,6 +22,8 @@
         (mkRustcFeatureArgs crateFeatures)
       ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
         "--target" (rust.toRustTargetSpec stdenv.hostPlatform)
+      ] ++ lib.optionals (needUnstableCLI dependencies) [
+        "-Z" "unstable-options"
       ] ++ extraRustcOpts
       # since rustc 1.42 the "proc_macro" crate is part of the default crate prelude
       # https://github.com/rust-lang/cargo/commit/4d64eb99a4#diff-7f98585dbf9d30aa100c8318e2c77e79R1021-R1022

pkgs/build-support/rust/build-rust-crate/default.nix

@@ -34,15 +34,25 @@ let
             )
           else
             extern;
-        in (if lib.any (x: x == "lib" || x == "rlib") dep.crateType then
-           " --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}.rlib"
-         else
-           " --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}")
+          opts = lib.optionalString (dep.stdlib or false) "noprelude:";
+          filename = if lib.any (x: x == "lib" || x == "rlib") dep.crateType
+            then "${dep.metadata}.rlib"
+            else "${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}";
+        in " --extern ${opts}${name}=${dep.lib}/lib/lib${extern}-${filename}"
       ) dependencies;
 
    # Create feature arguments for rustc.
    mkRustcFeatureArgs = lib.concatMapStringsSep " " (f: ''--cfg feature=\"${f}\"'');
 
+   # Whether we need to use unstable command line flags
+   #
+   # Currently just needed for standard library dependencies, which have a
+   # special "noprelude:" modifier. If in later versions of Rust this is
+   # stabilized we can account for that here, too, so we don't opt into
+   # instability unnecessarily.
+   needUnstableCLI = dependencies:
+     lib.any (dep: dep.stdlib or false) dependencies;
+
    inherit (import ./log.nix { inherit lib; }) noisily echo_colored;
 
    configureCrate = import ./configure-crate.nix {
@@ -50,7 +60,7 @@ let
    };
 
    buildCrate = import ./build-crate.nix {
-     inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs rust;
+     inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI rust;
    };
 
    installCrate = import ./install-crate.nix { inherit stdenv; };
@@ -247,9 +257,14 @@ stdenv.mkDerivation (rec {
 
     # Create a list of features that are enabled by the crate itself and
     # through the features argument of buildRustCrate. Exclude features
-    # with a forward slash, since they are passed through to dependencies.
+    # with a forward slash, since they are passed through to dependencies,
+    # and dep: features, since they're internal-only and do nothing except
+    # enable optional dependencies.
     crateFeatures = lib.optionals (crate ? features)
-      (builtins.filter (f: !lib.hasInfix "/" f) (crate.features ++ features));
+      (builtins.filter
+        (f: !(lib.hasInfix "/" f || lib.hasPrefix "dep:" f))
+        (crate.features ++ features)
+      );
 
     libName = if crate ? libName then crate.libName else crate.crateName;
     libPath = if crate ? libPath then crate.libPath else "";

pkgs/build-support/rust/lib/default.nix

@@ -3,12 +3,14 @@
 rec {
   # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch
   toTargetArch = platform:
-    if platform.isAarch32 then "arm"
+    /**/ if platform ? rustc.platform then platform.rustc.platform.arch
+    else if platform.isAarch32 then "arm"
     else platform.parsed.cpu.name;
 
   # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os
   toTargetOs = platform:
-    if platform.isDarwin then "macos"
+    /**/ if platform ? rustc.platform then platform.rustc.platform.os or "none"
+    else if platform.isDarwin then "macos"
     else platform.parsed.kernel.name;
 
   # Returns the name of the rust target, even if it is custom. Adjustments are
@@ -31,7 +33,7 @@ rec {
   # Returns the name of the rust target if it is standard, or the json file
   # containing the custom target spec.
   toRustTargetSpec = platform:
-    if (platform.rustc or {}) ? platform
+    if platform ? rustc.platform
     then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform)
     else toRustTarget platform;
 }

pkgs/build-support/setup-hooks/auto-patchelf.sh

@@ -37,7 +37,6 @@ isExecutable() {
 declare -Ag autoPatchelfCachedDepsAssoc
 declare -ag autoPatchelfCachedDeps
 
-
 addToDepCache() {
     if [[ ${autoPatchelfCachedDepsAssoc[$1]+f} ]]; then return; fi
 
@@ -53,25 +52,113 @@ declare -gi depCacheInitialised=0
 declare -gi doneRecursiveSearch=0
 declare -g foundDependency
 
-getDepsFromSo() {
-    ldd "$1" 2> /dev/null | sed -n -e 's/[^=]*=> *\(.\+\) \+([^)]*)$/\1/p'
+getDepsFromElfBinary() {
+    # NOTE: This does not use runPatchelf because it may encounter non-ELF
+    # files. Caller is expected to check the return code if needed.
+    patchelf --print-needed "$1" 2> /dev/null
+}
+
+getRpathFromElfBinary() {
+    # NOTE: This does not use runPatchelf because it may encounter non-ELF
+    # files. Caller is expected to check the return code if needed.
+    local rpath
+    IFS=':' read -ra rpath < <(patchelf --print-rpath "$1" 2> /dev/null) || return $?
+
+    printf "%s\n" "${rpath[@]}"
+}
+
+populateCacheForDep() {
+    local so="$1"
+    local rpath found
+    rpath="$(getRpathFromElfBinary "$so")" || return 1
+
+    for found in $(getDepsFromElfBinary "$so"); do
+        local rpathElem
+        for rpathElem in $rpath; do
+            # Ignore empty element or $ORIGIN magic variable which should be
+            # deterministically resolved by adding this package's library
+            # files early anyway.
+            #
+            # shellcheck disable=SC2016
+            # (Expressions don't expand in single quotes, use double quotes for
+            # that.)
+            if [[ -z "$rpathElem" || "$rpathElem" == *'$ORIGIN'* ]]; then
+                continue
+            fi
+
+            local soname="${found%.so*}"
+            local foundso=
+            for foundso in "$rpathElem/$soname".so*; do
+                addToDepCache "$foundso"
+            done
+
+            # Found in this element of the rpath, no need to check others.
+            if [ -n "$foundso" ]; then
+                break
+            fi
+        done
+    done
+
+    # Not found in any rpath element.
+    return 1
 }
 
 populateCacheWithRecursiveDeps() {
-    local so found foundso
-    for so in "${autoPatchelfCachedDeps[@]}"; do
-        for found in $(getDepsFromSo "$so"); do
-            local base="${found##*/}"
-            local soname="${base%.so*}"
-            for foundso in "${found%/*}/$soname".so*; do
-                addToDepCache "$foundso"
-            done
-        done
+    # Dependencies may add more to the end of this array, so we use a counter
+    # with while instead of a regular for loop here.
+    local -i i=0
+    while [ $i -lt ${#autoPatchelfCachedDeps[@]} ]; do
+        populateCacheForDep "${autoPatchelfCachedDeps[$i]}"
+        i=$i+1
     done
 }
 
-getSoArch() {
-    objdump -f "$1" | sed -ne 's/^architecture: *\([^,]\+\).*/\1/p'
+getBinArch() {
+    $OBJDUMP -f "$1" 2> /dev/null | sed -ne 's/^architecture: *\([^,]\+\).*/\1/p'
+}
+
+# Returns the specific OS ABI for an ELF file in the format produced by
+# readelf(1), like "UNIX - System V" or "UNIX - GNU".
+getBinOsabi() {
+    $READELF -h "$1" 2> /dev/null | sed -ne 's/^[ \t]*OS\/ABI:[ \t]*\(.*\)/\1/p'
+}
+
+# Tests whether two OS ABIs are compatible, taking into account the generally
+# accepted compatibility of SVR4 ABI with other ABIs.
+areBinOsabisCompatible() {
+    local wanted="$1"
+    local got="$2"
+
+    if [[ -z "$wanted" || -z "$got" ]]; then
+        # One of the types couldn't be detected, so as a fallback we'll assume
+        # they're compatible.
+        return 0
+    fi
+
+    # Generally speaking, the base ABI (0x00), which is represented by
+    # readelf(1) as "UNIX - System V", indicates broad compatibility with other
+    # ABIs.
+    #
+    # TODO: This isn't always true. For example, some OSes embed ABI
+    # compatibility into SHT_NOTE sections like .note.tag and .note.ABI-tag.
+    # It would be prudent to add these to the detection logic to produce better
+    # ABI information.
+    if [[ "$wanted" == "UNIX - System V" ]]; then
+        return 0
+    fi
+
+    # Similarly here, we should be able to link against a superset of features,
+    # so even if the target has another ABI, this should be fine.
+    if [[ "$got" == "UNIX - System V" ]]; then
+        return 0
+    fi
+
+    # Otherwise, we simply return whether the ABIs are identical.
+    if [[ "$wanted" == "$got" ]]; then
+        return 0
+    fi
+
+    return 1
 }
 
 # NOTE: If you want to use this function outside of the autoPatchelf function,
@@ -82,6 +169,7 @@ getSoArch() {
 findDependency() {
     local filename="$1"
     local arch="$2"
+    local osabi="$3"
     local lib dep
 
     if [ $depCacheInitialised -eq 0 ]; then
@@ -93,7 +181,7 @@ findDependency() {
 
     for dep in "${autoPatchelfCachedDeps[@]}"; do
         if [ "$filename" = "${dep##*/}" ]; then
-            if [ "$(getSoArch "$dep")" = "$arch" ]; then
+            if [ "$(getBinArch "$dep")" = "$arch" ] && areBinOsabisCompatible "$osabi" "$(getBinOsabi "$dep")"; then
                 foundDependency="$dep"
                 return 0
             fi
@@ -117,7 +205,24 @@ autoPatchelfFile() {
     local dep rpath="" toPatch="$1"
 
     local interpreter
-    interpreter="$(< "$NIX_CC/nix-support/dynamic-linker")"
+    interpreter="$(< "$NIX_BINTOOLS/nix-support/dynamic-linker")"
+
+    local interpreterArch interpreterOsabi toPatchArch toPatchOsabi
+    interpreterArch="$(getBinArch "$interpreter")"
+    interpreterOsabi="$(getBinOsabi "$interpreter")"
+    toPatchArch="$(getBinArch "$toPatch")"
+    toPatchOsabi="$(getBinOsabi "$toPatch")"
+
+    if [ "$interpreterArch" != "$toPatchArch" ]; then
+        # Our target architecture is different than this file's architecture,
+        # so skip it.
+        echo "skipping $toPatch because its architecture ($toPatchArch) differs from target ($interpreterArch)" >&2
+        return 0
+    elif ! areBinOsabisCompatible "$interpreterOsabi" "$toPatchOsabi"; then
+        echo "skipping $toPatch because its OS ABI ($toPatchOsabi) is not compatible with target ($interpreterOsabi)" >&2
+        return 0
+    fi
+
     if isExecutable "$toPatch"; then
         runPatchelf --set-interpreter "$interpreter" "$toPatch"
         # shellcheck disable=SC2154
@@ -129,27 +234,34 @@ autoPatchelfFile() {
         fi
     fi
 
+    local libcLib
+    libcLib="$(< "$NIX_BINTOOLS/nix-support/orig-libc")/lib"
+
     echo "searching for dependencies of $toPatch" >&2
 
-    # We're going to find all dependencies based on ldd output, so we need to
-    # clear the RPATH first.
-    runPatchelf --remove-rpath "$toPatch"
-
-    # If the file is not a dynamic executable, ldd/sed will fail,
-    # in which case we return, since there is nothing left to do.
     local missing
-    missing="$(
-        ldd "$toPatch" 2> /dev/null | \
-            sed -n -e 's/^[\t ]*\([^ ]\+\) => not found.*/\1/p'
-    )" || return 0
+    missing="$(getDepsFromElfBinary "$toPatch")" || return 0
 
     # This ensures that we get the output of all missing dependencies instead
     # of failing at the first one, because it's more useful when working on a
     # new package where you don't yet know its dependencies.
 
     for dep in $missing; do
+        if [[ "$dep" == /* ]]; then
+            # This is an absolute path. If it exists, just use it. Otherwise,
+            # we probably want this to produce an error when checked (because
+            # just updating the rpath won't satisfy it).
+            if [ -f "$dep" ]; then
+                continue
+            fi
+        elif [ -f "$libcLib/$dep" ]; then
+            # This library exists in libc, and will be correctly resolved by
+            # the linker.
+            continue
+        fi
+
         echo -n "  $dep -> " >&2
-        if findDependency "$dep" "$(getSoArch "$toPatch")"; then
+        if findDependency "$dep" "$toPatchArch" "$toPatchOsabi"; then
             rpath="$rpath${rpath:+:}${foundDependency%/*}"
             echo "found: $foundDependency" >&2
         else
@@ -184,7 +296,7 @@ addAutoPatchelfSearchPath() {
     done
 
     while IFS= read -r -d '' file; do
-    addToDepCache "$file"
+        addToDepCache "$file"
     done <  <(find "$@" "${findOpts[@]}" \! -type d \
             \( -name '*.so' -o -name '*.so.*' \) -print0)
 }
@@ -220,10 +332,10 @@ autoPatchelf() {
       segmentHeaders="$(LANG=C $READELF -l "$file")"
       # Skip if the ELF file doesn't have segment headers (eg. object files).
       # not using grep -q, because it can cause Broken pipe
-      [ -n "$(echo "$segmentHeaders" | grep '^Program Headers:')" ] || continue
+      grep -q '^Program Headers:' <<<"$segmentHeaders" || continue
       if isExecutable "$file"; then
           # Skip if the executable is statically linked.
-          [ -n "$(echo "$segmentHeaders" | grep "^ *INTERP\\>")" ] || continue
+          grep -q "^ *INTERP\\>" <<<"$segmentHeaders" || continue
       fi
       # Jump file if patchelf is unable to parse it
       # Some programs contain binary blobs for testing,
@@ -255,6 +367,9 @@ autoPatchelf() {
 # So what we do here is basically run in postFixup and emulate the same
 # behaviour as fixupOutputHooks because the setup hook for patchelf is run in
 # fixupOutput and the postFixup hook runs later.
+#
+# shellcheck disable=SC2016
+# (Expressions don't expand in single quotes, use double quotes for that.)
 postFixupHooks+=('
     if [ -z "${dontAutoPatchelf-}" ]; then
         autoPatchelf -- $(for output in $outputs; do

pkgs/development/compilers/llvm/12/compiler-rt/default.nix

@@ -30,7 +30,7 @@ stdenv.mkDerivation {
     "-DCOMPILER_RT_BUILD_XRAY=OFF"
     "-DCOMPILER_RT_BUILD_LIBFUZZER=OFF"
     "-DCOMPILER_RT_BUILD_PROFILE=OFF"
-  ] ++ lib.optionals ((useLLVM || bareMetal) && !haveLibc) [
+  ] ++ lib.optionals ((useLLVM && !haveLibc) || bareMetal) [
     "-DCMAKE_C_COMPILER_WORKS=ON"
     "-DCMAKE_CXX_COMPILER_WORKS=ON"
     "-DCOMPILER_RT_BAREMETAL_BUILD=ON"

pkgs/development/compilers/llvm/13/compiler-rt/default.nix

@@ -32,7 +32,7 @@ stdenv.mkDerivation {
     "-DCOMPILER_RT_BUILD_XRAY=OFF"
     "-DCOMPILER_RT_BUILD_LIBFUZZER=OFF"
     "-DCOMPILER_RT_BUILD_PROFILE=OFF"
-  ] ++ lib.optionals ((useLLVM || bareMetal) && !haveLibc) [
+  ] ++ lib.optionals ((useLLVM && !haveLibc) || bareMetal) [
     "-DCMAKE_C_COMPILER_WORKS=ON"
     "-DCMAKE_CXX_COMPILER_WORKS=ON"
     "-DCOMPILER_RT_BAREMETAL_BUILD=ON"

pkgs/top-level/all-packages.nix

@@ -169,7 +169,8 @@ in
     autoconf = autoconf269;
   };
 
-  autoPatchelfHook = makeSetupHook { name = "auto-patchelf-hook"; }
+  autoPatchelfHook = makeSetupHook
+    { name = "auto-patchelf-hook"; deps = [ bintools ]; }
     ../build-support/setup-hooks/auto-patchelf.sh;
 
   appimageTools = callPackage ../build-support/appimage {