Merge pull request #83601 from andir/20.03-buildRustCrate-fixes

[20.03] buildRustCrate fixes backports
This commit is contained in:
Florian Klink
2020-04-04 01:43:32 +02:00
committed by GitHub
7 changed files with 440 additions and 71 deletions

View File

@@ -1,4 +1,4 @@
{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs, rust }:
{ lib, stdenv, mkRustcDepArgs, rust }:
{ crateName,
dependencies,
crateFeatures, crateRenames, libName, release, libPath,
@@ -11,6 +11,7 @@
baseRustcOpts =
[(if release then "-C opt-level=3" else "-C debuginfo=2")]
++ ["-C codegen-units=$NIX_BUILD_CORES"]
++ ["--remap-path-prefix=$NIX_BUILD_TOP=/" ]
++ [(mkRustcDepArgs dependencies crateRenames)]
++ [crateFeatures]
++ extraRustcOpts
@@ -34,16 +35,13 @@
build_bin = if buildTests then "build_bin_test" else "build_bin";
in ''
runHook preBuild
${echo_build_heading colors}
${noisily colors verbose}
# configure & source common build functions
LIB_RUSTC_OPTS="${libRustcOpts}"
BIN_RUSTC_OPTS="${binRustcOpts}"
LIB_EXT="${stdenv.hostPlatform.extensions.sharedLibrary}"
LIB_PATH="${libPath}"
LIB_NAME="${libName}"
source ${./lib.sh}
CRATE_NAME='${lib.replaceStrings ["-"] ["_"] libName}'

View File

@@ -1,4 +1,4 @@
{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs }:
{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs }:
{ build
, buildDependencies
, colors
@@ -31,42 +31,88 @@ let version_ = lib.splitString "-" crateVersion;
completeDepsDir = lib.concatStringsSep " " completeDeps;
completeBuildDepsDir = lib.concatStringsSep " " completeBuildDeps;
in ''
cd ${workspace_member}
runHook preConfigure
${echo_build_heading colors}
${echo_colored colors}
${noisily colors verbose}
source ${./lib.sh}
${lib.optionalString (workspace_member != null) ''
noisily cd "${workspace_member}"
''}
${lib.optionalString (workspace_member == null) ''
echo_colored "Searching for matching Cargo.toml (${crateName})"
local cargo_toml_dir=$(matching_cargo_toml_dir "${crateName}")
if [ -z "$cargo_toml_dir" ]; then
echo_error "ERROR configuring ${crateName}: No matching Cargo.toml in $(pwd) found." >&2
exit 23
fi
noisily cd "$cargo_toml_dir"
''}
runHook preConfigure
symlink_dependency() {
# $1 is the nix-store path of a dependency
# $2 is the target path
i=$1
ln -s -f $i/lib/*.rlib $2 #*/
ln -s -f $i/lib/*.so $i/lib/*.dylib $2 #*/
if [ -e "$i/lib/link" ]; then
cat $i/lib/link >> target/link
cat $i/lib/link >> target/link.final
fi
if [ -e $i/env ]; then
source $i/env
fi
}
# The following steps set up the dependencies of the crate. Two
# kinds of dependencies are distinguished: build dependencies
# (used by the build script) and crate dependencies. For each
# dependency we have to:
#
# - Make its Rust library available to rustc. This is done by
# symlinking all library dependencies into a directory that
# can be provided to rustc.
# - Accumulate linking flags. These flags are largely used for
# linking native libraries.
#
# The crate link flags are added to the `link` and `link.final`
# files. The `link` file is used for linkage in the current
# crate. The `link.final` file will be copied to the output and can
# be used by downstream crates to get the linker flags of this
# crate.
mkdir -p target/{deps,lib,build,buildDeps}
chmod uga+w target -R
echo ${extraLinkFlags} > target/link
echo ${extraLinkFlags} > target/link.final
# Prepare crate dependencies
for i in ${completeDepsDir}; do
symlink_dependency $i target/deps
if [ -e "$i/lib/link" ]; then
cat $i/lib/link >> target/link
cat $i/lib/link >> target/link.final
fi
done
# Prepare crate build dependencies that are used for the build script.
for i in ${completeBuildDepsDir}; do
symlink_dependency $i target/buildDeps
symlink_dependency $i target/buildDeps
if [ -e "$i/lib/link" ]; then
cat $i/lib/link >> target/link.build
fi
done
if [[ -e target/link ]]; then
sort -u target/link > target/link.sorted
mv target/link.sorted target/link
sort -u target/link.final > target/link.final.sorted
mv target/link.final.sorted target/link.final
tr '\n' ' ' < target/link > target/link_
# Remove duplicate linker flags from the build dependencies.
if [[ -e target/link.build ]]; then
sort -uo target/link.build target/link.build
fi
# Remove duplicate linker flags from the dependencies.
sort -uo target/link target/link
tr '\n' ' ' < target/link > target/link_
# Remove duplicate linker flags from the that are written
# to the derivation's output.
sort -uo target/link.final target/link.final
EXTRA_BUILD=""
BUILD_OUT_DIR=""
export CARGO_PKG_NAME=${crateName}
@@ -105,15 +151,14 @@ in ''
elif [[ -e "build.rs" ]]; then
BUILD="build.rs"
fi
# Compile and run the build script, when available.
if [[ ! -z "$BUILD" ]] ; then
echo_build_heading "$BUILD" ${libName}
mkdir -p target/build/${crateName}
EXTRA_BUILD_FLAGS=""
if [ -e target/link_ ]; then
EXTRA_BUILD_FLAGS=$(cat target/link_)
fi
if [ -e target/link.build ]; then
EXTRA_BUILD_FLAGS="$EXTRA_BUILD_FLAGS $(cat target/link.build)"
EXTRA_BUILD_FLAGS="$EXTRA_BUILD_FLAGS $(tr '\n' ' ' < target/link.build)"
fi
noisily rustc --crate-name build_script_build $BUILD --crate-type bin ${rustcOpts} \
${crateFeatures} --out-dir target/build/${crateName} --emit=dep-info,link \
@@ -127,7 +172,7 @@ in ''
set +e
EXTRA_BUILD=$(sed -n "s/^cargo:rustc-flags=\(.*\)/\1/p" target/build/${crateName}.opt | tr '\n' ' ' | sort -u)
EXTRA_FEATURES=$(sed -n "s/^cargo:rustc-cfg=\(.*\)/--cfg \1/p" target/build/${crateName}.opt | tr '\n' ' ')
EXTRA_LINK=$(sed -n "s/^cargo:rustc-link-lib=\(.*\)/\1/p" target/build/${crateName}.opt | tr '\n' ' ' | sort -u)
EXTRA_LINK=$(sed -n "s/^cargo:rustc-link-lib=\(.*\)/\1/p" target/build/${crateName}.opt | tr '\n' ' ')
EXTRA_LINK_SEARCH=$(sed -n "s/^cargo:rustc-link-search=\(.*\)/\1/p" target/build/${crateName}.opt | tr '\n' ' ' | sort -u)
for env in $(sed -n "s/^cargo:rustc-env=\(.*\)/\1/p" target/build/${crateName}.opt); do

View File

@@ -4,7 +4,7 @@
# This can be useful for deploying packages with NixOps, and to share
# binary dependencies between projects.
{ lib, stdenv, defaultCrateOverrides, fetchCrate, rustc, rust }:
{ lib, stdenv, defaultCrateOverrides, fetchCrate, rustc, rust, cargo, jq }:
let
# This doesn't appear to be officially documented anywhere yet.
@@ -29,14 +29,14 @@ let
" --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}")
) dependencies;
inherit (import ./log.nix { inherit lib; }) noisily echo_build_heading;
inherit (import ./log.nix { inherit lib; }) noisily echo_colored;
configureCrate = import ./configure-crate.nix {
inherit lib stdenv echo_build_heading noisily mkRustcDepArgs;
inherit lib stdenv echo_colored noisily mkRustcDepArgs;
};
buildCrate = import ./build-crate.nix {
inherit lib stdenv echo_build_heading noisily mkRustcDepArgs rust;
inherit lib stdenv mkRustcDepArgs rust;
};
installCrate = import ./install-crate.nix { inherit stdenv; };
@@ -62,9 +62,6 @@ let crate = crate_ // (lib.attrByPath [ crate_.crateName ] (attr: {}) crateOverr
extraRustcOpts_ = extraRustcOpts;
buildTests_ = buildTests;
# take a list of crates that we depend on and override them to fit our overrides, rustc, release, …
makeDependencies = map (dep: lib.getLib (dep.override { inherit release verbose crateOverrides; }));
# crate2nix has a hack for the old bash based build script that did split
# entries at `,`. No we have to work around that hack.
# https://github.com/kolloch/crate2nix/blame/5b19c1b14e1b0e5522c3e44e300d0b332dc939e7/crate2nix/templates/build.nix.tera#L89
@@ -91,10 +88,10 @@ stdenv.mkDerivation (rec {
src = crate.src or (fetchCrate { inherit (crate) crateName version sha256; });
name = "rust_${crate.crateName}-${crate.version}${lib.optionalString buildTests_ "-test"}";
depsBuildBuild = [ rust stdenv.cc ];
depsBuildBuild = [ rust stdenv.cc cargo jq ];
buildInputs = (crate.buildInputs or []) ++ buildInputs_;
dependencies = makeDependencies dependencies_;
buildDependencies = makeDependencies buildDependencies_;
dependencies = map lib.getLib dependencies_;
buildDependencies = map lib.getLib buildDependencies_;
completeDeps = lib.unique (dependencies ++ lib.concatMap (dep: dep.completeDeps) dependencies);
completeBuildDeps = lib.unique (
@@ -103,7 +100,7 @@ stdenv.mkDerivation (rec {
);
crateFeatures = lib.optionalString (crate ? features)
(lib.concatMapStringsSep " " (f: "--cfg feature=\\\"${f}\\\"") (crate.features ++ features));
(lib.concatMapStringsSep " " (f: ''--cfg feature=\"${f}\"'') (crate.features ++ features));
libName = if crate ? libName then crate.libName else crate.crateName;
libPath = if crate ? libPath then crate.libPath else "";
@@ -117,6 +114,8 @@ stdenv.mkDerivation (rec {
in lib.substring 0 10 hashedMetadata;
build = crate.build or "";
# Either set to a concrete sub path to the crate root
# or use `null` for auto-detect.
workspace_member = crate.workspace_member or ".";
crateVersion = crate.version;
crateDescription = crate.description or "";

View File

@@ -14,7 +14,7 @@ if !buildTests then ''
fi
if [[ "$(ls -A target/lib)" ]]; then
mkdir -p $lib/lib
cp target/lib/* $lib/lib #*/
cp -r target/lib/* $lib/lib #*/
for library in $lib/lib/*.so $lib/lib/*.dylib; do #*/
ln -s $library $(echo $library | sed -e "s/-${metadata}//")
done
@@ -26,7 +26,7 @@ if !buildTests then ''
if [[ -d target/bin ]]; then
if [[ "$(ls -A target/bin)" ]]; then
mkdir -p $out/bin
cp -P target/bin/* $out/bin # */
cp -rP target/bin/* $out/bin # */
fi
fi
runHook postInstall

View File

@@ -1,3 +1,11 @@
echo_build_heading() {
if (( $# == 1 )); then
echo_colored "Building $1"
else
echo_colored "Building $1 ($2)"
fi
}
build_lib() {
lib_src=$1
echo_build_heading $lib_src ${libName}
@@ -6,7 +14,6 @@ build_lib() {
--crate-name $CRATE_NAME \
$lib_src \
--out-dir target/lib \
--emit=dep-info,link \
-L dependency=target/deps \
--cap-lints allow \
$LIB_RUSTC_OPTS \
@@ -37,7 +44,6 @@ build_bin() {
--crate-type bin \
$BIN_RUSTC_OPTS \
--out-dir target/bin \
--emit=dep-info,link \
-L dependency=target/deps \
$LINK \
$EXTRA_LIB \
@@ -71,6 +77,7 @@ build_bin_test_file() {
build_bin_test "$derived_crate_name" "$file"
}
# Add additional link options that were provided by the build script.
setup_link_paths() {
EXTRA_LIB=""
if [[ -e target/link_ ]]; then
@@ -96,11 +103,6 @@ setup_link_paths() {
done
if [[ -e target/link ]]; then
sort -u target/link.final > target/link.final.sorted
mv target/link.final.sorted target/link.final
sort -u target/link > target/link.sorted
mv target/link.sorted target/link
tr '\n' ' ' < target/link > target/link_
LINK=$(cat target/link_)
fi
@@ -132,7 +134,41 @@ search_for_bin_path() {
done
if [[ -z "$BIN_PATH" ]]; then
echo "failed to find file for binary target: $BIN_NAME" >&2
echo_error "ERROR: failed to find file for binary target: $BIN_NAME" >&2
exit 1
fi
}
# Extracts cargo_toml_path of the matching crate.
matching_cargo_toml_path() {
local manifest_path="$1"
local expected_crate_name="$2"
# If the Cargo.toml is not a workspace root,
# it will only contain one package in ".packages"
# because "--no-deps" suppressed dependency resolution.
#
# But to make it more general, we search for a matching
# crate in all packages and use the manifest path that
# is referenced there.
cargo metadata --no-deps --format-version 1 \
--manifest-path "$manifest_path" \
| jq -r '.packages[]
| select( .name == "'$expected_crate_name'")
| .manifest_path'
}
# Find a Cargo.toml in the current or any sub directory
# with a matching crate name.
matching_cargo_toml_dir() {
local expected_crate_name="$1"
find -L -name Cargo.toml | sort | while read manifest_path; do
echo "...checking manifest_path $manifest_path" >&2
local matching_path="$(matching_cargo_toml_path "$manifest_path" "$expected_crate_name")"
if [ -n "${matching_path}" ]; then
echo "$(dirname $matching_path)"
break
fi
done
}

View File

@@ -1,30 +1,56 @@
{ lib }:
{
echo_build_heading = colors: ''
echo_build_heading() {
start=""
end=""
${lib.optionalString (colors == "always") ''
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
''}
if (( $# == 1 )); then
echo "$start""Building $1""$end"
else
echo "$start""Building $1 ($2)""$end"
fi
let echo_colored_body = start_escape:
# Body of a function that behaves like "echo" but
# has the output colored by the given start_escape
# sequence. E.g.
#
# * echo_x "Building ..."
# * echo_x -n "Running "
#
# This is more complicated than apparent at first sight
# because:
# * The color markers and the text must be print
# in the same echo statement. Otherise, other
# intermingled text from concurrent builds will
# be colored as well.
# * We need to preserve the trailing newline of the
# echo if and only if it is present. Bash likes
# to strip those if we capture the output of echo
# in a variable.
# * Leading "-" will be interpreted by test as an
# option for itself. Therefore, we prefix it with
# an x in `[[ "x$1" =~ ^x- ]]`.
''
local echo_args="";
while [[ "x$1" =~ ^x- ]]; do
echo_args+=" $1"
shift
done
local start_escape="$(printf '${start_escape}')"
local reset="$(printf '\033[0m')"
echo $echo_args $start_escape"$@"$reset
'';
echo_conditional_colored_body = colors: start_escape:
if colors == "always"
then (echo_colored_body start_escape)
else ''echo "$@"'';
in {
echo_colored = colors: ''
echo_colored() {
${echo_conditional_colored_body colors ''\033[0;1;32m''}
}
'';
echo_error() {
${echo_conditional_colored_body colors ''\033[0;1;31m''}
}
'';
noisily = colors: verbose: ''
noisily() {
start=""
end=""
${lib.optionalString (colors == "always") ''
start="$(printf '\033[0;1;32m')" #set bold, and set green.
end="$(printf '\033[0m')" #returns to "normal"
''}
${lib.optionalString verbose ''
echo -n "$start"Running "$end"
echo_colored -n "Running "
echo $@
''}
$@

View File

@@ -1,4 +1,14 @@
{ lib, buildRustCrate, runCommand, writeTextFile, symlinkJoin, callPackage, releaseTools }:
{ lib
, buildRustCrate
, callPackage
, releaseTools
, runCommand
, runCommandCC
, stdenv
, symlinkJoin
, writeTextFile
}:
let
mkCrate = args: let
p = {
@@ -8,6 +18,14 @@ let
} // args;
in buildRustCrate p;
mkCargoToml =
{ name, crateVersion ? "0.1.0", path ? "Cargo.toml" }:
mkFile path ''
[package]
name = ${builtins.toJSON name}
version = ${builtins.toJSON crateVersion}
'';
mkFile = destination: text: writeTextFile {
name = "src";
destination = "/${destination}";
@@ -86,10 +104,62 @@ let
''
);
/* Returns a derivation that asserts that the crate specified by `crateArgs`
has the specified files as output.
`name` is used as part of the derivation name that performs the checking.
`crateArgs` is passed to `mkCrate` to build the crate with `buildRustCrate`.
`expectedFiles` contains a list of expected file paths in the output. E.g.
`[ "./bin/my_binary" ]`.
`output` specifies the name of the output to use. By default, the default
output is used but e.g. `output = "lib";` will cause the lib output
to be checked instead. You do not need to specify any directories.
*/
assertOutputs = { name, crateArgs, expectedFiles, output? null }:
assert (builtins.isString name);
assert (builtins.isAttrs crateArgs);
assert (builtins.isList expectedFiles);
let
crate = mkCrate (builtins.removeAttrs crateArgs ["expectedTestOutput"]);
crateOutput = if output == null then crate else crate."${output}";
expectedFilesFile = writeTextFile {
name = "expected-files-${name}";
text =
let sorted = builtins.sort (a: b: a<b) expectedFiles;
concatenated = builtins.concatStringsSep "\n" sorted;
in "${concatenated}\n";
};
in
runCommand "assert-outputs-${name}" {
} ''
local actualFiles=$(mktemp)
cd "${crateOutput}"
find . -type f | sort >$actualFiles
diff -q ${expectedFilesFile} $actualFiles >/dev/null || {
echo -e "\033[0;1;31mERROR: Difference in expected output files in ${crateOutput} \033[0m" >&2
echo === Got:
sed -e 's/^/ /' $actualFiles
echo === Expected:
sed -e 's/^/ /' ${expectedFilesFile}
echo === Diff:
diff -u ${expectedFilesFile} $actualFiles |\
tail -n +3 |\
sed -e 's/^/ /'
exit 1
}
touch $out
''
;
in rec {
tests = let
cases = {
cases = rec {
libPath = { libPath = "src/my_lib.rs"; src = mkLib "src/my_lib.rs"; };
srcLib = { src = mkLib "src/lib.rs"; };
@@ -199,6 +269,36 @@ let
})
];
};
buildScriptDeps = let
depCrate = boolVal: mkCrate {
crateName = "bar";
src = mkFile "src/lib.rs" ''
pub const baz: bool = ${boolVal};
'';
};
in {
crateName = "foo";
src = symlinkJoin {
name = "build-script-and-main";
paths = [
(mkFile "src/main.rs" ''
extern crate bar;
#[cfg(test)]
#[test]
fn baz_false() { assert!(!bar::baz); }
fn main() { }
'')
(mkFile "build.rs" ''
extern crate bar;
fn main() { assert!(bar::baz); }
'')
];
};
buildDependencies = [ (depCrate "true") ];
dependencies = [ (depCrate "false") ];
buildTests = true;
expectedTestOutputs = [ "test baz_false ... ok" ];
};
# Regression test for https://github.com/NixOS/nixpkgs/issues/74071
# Whenevever a build.rs file is generating files those should not be overlayed onto the actual source dir
buildRsOutDirOverlay = {
@@ -220,9 +320,174 @@ let
];
};
};
# Regression test for https://github.com/NixOS/nixpkgs/pull/83379
# link flag order should be preserved
linkOrder = {
src = symlinkJoin {
name = "buildrs-out-dir-overlay";
paths = [
(mkFile "build.rs" ''
fn main() {
// in the other order, linkage will fail
println!("cargo:rustc-link-lib=b");
println!("cargo:rustc-link-lib=a");
}
'')
(mkFile "src/main.rs" ''
extern "C" {
fn hello_world();
}
fn main() {
unsafe {
hello_world();
}
}
'')
];
};
buildInputs = let
compile = name: text: let
src = writeTextFile {
name = "${name}-src.c";
inherit text;
};
in runCommandCC name {} ''
mkdir -p $out/lib
# Note: On darwin (which defaults to clang) we have to add
# `-undefined dynamic_lookup` as otherwise the compilation fails.
cc -shared \
${lib.optionalString stdenv.isDarwin "-undefined dynamic_lookup"} \
-o $out/lib/${name}${stdenv.hostPlatform.extensions.sharedLibrary} ${src}
'';
b = compile "libb" ''
#include <stdio.h>
void hello();
void hello_world() {
hello();
printf(" world!\n");
}
'';
a = compile "liba" ''
#include <stdio.h>
void hello() {
printf("hello");
}
'';
in [ a b ];
};
rustCargoTomlInSubDir = {
# The "workspace_member" can be set to the sub directory with the crate to build.
# By default ".", meaning the top level directory is assumed.
# Using null will trigger a search.
workspace_member = null;
src = symlinkJoin rec {
name = "find-cargo-toml";
paths = [
(mkCargoToml { name = "ignoreMe"; })
(mkTestFileWithMain "src/main.rs" "ignore_main")
(mkCargoToml { name = "rustCargoTomlInSubDir"; path = "subdir/Cargo.toml"; })
(mkTestFileWithMain "subdir/src/main.rs" "src_main")
(mkTestFile "subdir/tests/foo/main.rs" "tests_foo")
(mkTestFile "subdir/tests/bar/main.rs" "tests_bar")
];
};
buildTests = true;
expectedTestOutputs = [
"test src_main ... ok"
"test tests_foo ... ok"
"test tests_bar ... ok"
];
};
rustCargoTomlInTopDir =
let
withoutCargoTomlSearch = builtins.removeAttrs rustCargoTomlInSubDir [ "workspace_member" ];
in
withoutCargoTomlSearch // {
expectedTestOutputs = [
"test ignore_main ... ok"
];
};
};
brotliCrates = (callPackage ./brotli-crates.nix {});
in lib.mapAttrs (key: value: mkTest (value // lib.optionalAttrs (!value?crateName) { crateName = key; })) cases // {
tests = lib.mapAttrs (key: value: mkTest (value // lib.optionalAttrs (!value?crateName) { crateName = key; })) cases;
in tests // rec {
crateBinWithPathOutputs = assertOutputs {
name="crateBinWithPath";
crateArgs = {
crateBin = [{ name = "test_binary1"; path = "src/foobar.rs"; }];
src = mkBin "src/foobar.rs";
};
expectedFiles = [
"./bin/test_binary1"
];
};
crateBinWithPathOutputsDebug = assertOutputs {
name="crateBinWithPath";
crateArgs = {
release = false;
crateBin = [{ name = "test_binary1"; path = "src/foobar.rs"; }];
src = mkBin "src/foobar.rs";
};
expectedFiles = [
"./bin/test_binary1"
] ++ lib.optionals stdenv.isDarwin [
# On Darwin, the debug symbols are in a seperate directory.
"./bin/test_binary1.dSYM/Contents/Info.plist"
"./bin/test_binary1.dSYM/Contents/Resources/DWARF/test_binary1"
];
};
crateBinNoPath1Outputs = assertOutputs {
name="crateBinNoPath1";
crateArgs = {
crateBin = [{ name = "my-binary2"; }];
src = mkBin "src/my_binary2.rs";
};
expectedFiles = [
"./bin/my-binary2"
];
};
crateLibOutputs = assertOutputs {
name="crateLib";
output="lib";
crateArgs = {
libName = "test_lib";
type = [ "rlib" ];
libPath = "src/lib.rs";
src = mkLib "src/lib.rs";
};
expectedFiles = [
"./nix-support/propagated-build-inputs"
"./lib/libtest_lib-042a1fdbef.rlib"
"./lib/link"
];
};
crateLibOutputsDebug = assertOutputs {
name="crateLib";
output="lib";
crateArgs = {
release = false;
libName = "test_lib";
type = [ "rlib" ];
libPath = "src/lib.rs";
src = mkLib "src/lib.rs";
};
expectedFiles = [
"./nix-support/propagated-build-inputs"
"./lib/libtest_lib-042a1fdbef.rlib"
"./lib/link"
];
};
brotliTest = let
pkg = brotliCrates.brotli_2_5_0 {};
in runCommand "run-brotli-test-cmd" {