diff --git a/doc/builders/testers.chapter.md b/doc/builders/testers.chapter.md index ad1e1036d508..57f66a428482 100644 --- a/doc/builders/testers.chapter.md +++ b/doc/builders/testers.chapter.md @@ -35,6 +35,46 @@ passthru.tests.version = testers.testVersion { }; ``` +## `testBuildFailure` {#tester-testBuildFailure} + +Make sure that a build does not succeed. This is useful for testing testers. + +This returns a derivation with an override on the builder, with the following effects: + + - Fail the build when the original builder succeeds + - Move `$out` to `$out/result`, if it exists (assuming `out` is the default output) + - Save the build log to `$out/testBuildFailure.log` (same) + +Example: + +```nix +runCommand "example" { + failed = testers.testBuildFailure (runCommand "fail" {} '' + echo ok-ish >$out + echo failing though + exit 3 + ''); +} '' + grep -F 'ok-ish' $failed/result + grep -F 'failing though' $failed/testBuildFailure.log + [[ 3 = $(cat $failed/testBuildFailure.exit) ]] + touch $out +''; +``` + +While `testBuildFailure` is designed to keep changes to the original builder's +environment to a minimum, some small changes are inevitable. + + - The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted. + - `stdout` and `stderr` are a pipe instead of a tty. This could be improved. + - One or two extra processes are present in the sandbox during the original + builder's execution. + - The derivation and output hashes are different, but not unusual. + - The derivation includes a dependency on `buildPackages.bash` and + `expect-failure.sh`, which is built to include a transitive dependency on + `buildPackages.coreutils` and possibly more. These are not added to `PATH` + or any other environment variable, so they should be hard to observe. + ## `testEqualDerivation` {#tester-testEqualDerivation} Checks that two packages produce the exact same build instructions. diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 7244d3d38575..fd08e9c6c47f 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -1,6 +1,16 @@ -{ pkgs, lib, callPackage, runCommand, stdenv }: +{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }: # Documentation is in doc/builders/testers.chapter.md { + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure + # or doc/builders/testers.chapter.md + testBuildFailure = drv: drv.overrideAttrs (orig: { + builder = buildPackages.bash; + args = [ + (substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; }) + orig.realBuilder or stdenv.shell + ] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)]; + }); + testEqualDerivation = callPackage ./test-equal-derivation.nix { }; testVersion = diff --git a/pkgs/build-support/testers/expect-failure.sh b/pkgs/build-support/testers/expect-failure.sh new file mode 100644 index 000000000000..23e8698cf479 --- /dev/null +++ b/pkgs/build-support/testers/expect-failure.sh @@ -0,0 +1,62 @@ +# Run a builder, flip exit code, save log and fix outputs +# +# Sub-goals: +# - Delegate to another original builder passed via args +# - Save the build log to output for further checks +# - Make the derivation succeed if the original builder fails +# - Make the derivation fail if the original builder returns exit code 0 +# +# Requirements: +# This runs before, without and after stdenv. Do not modify the environment; +# especially not before invoking the original builder. For example, use +# "@" substitutions instead of PATH. +# Do not export any variables. + +# Stricter bash +set -eu + +# ------------------------ +# Run the original builder + +echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}" + +("$@" 2>&1) | @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \ + | while read ln; do + echo "original builder: $ln" + done + +r=${PIPESTATUS[0]} +if [[ $r = 0 ]]; then + echo "testBuildFailure: The builder did not fail, but a failure was expected!" + exit 1 +fi +echo "testBuildFailure: Original builder produced exit code: $r" + +# ----------------------------------------- +# Write the build log to the default output + +outs=( $outputs ) +defOut=${outs[0]} +defOutPath=${!defOut} + +if [[ ! -d $defOutPath ]]; then + if [[ -e $defOutPath ]]; then + @coreutils@/bin/mv $defOutPath $TMPDIR/out-node + @coreutils@/bin/mkdir $defOutPath + @coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result + fi +fi + +@coreutils@/bin/mkdir -p $defOutPath +@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log +echo $r >$defOutPath/testBuildFailure.exit + +# ------------------------------------------------------ +# Put empty directories in place for any missing outputs + +for outputName in ${outputs:-out}; do + outputPath="${!outputName}" + if [[ ! -e "${outputPath}" ]]; then + @coreutils@/bin/mkdir "${outputPath}"; + fi +done diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index 30e778cf652e..22869baae159 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -1,4 +1,4 @@ -{ testers, lib, pkgs, ... }: +{ testers, lib, pkgs, hello, runCommand, ... }: let pkgs-with-overlay = pkgs.extend(final: prev: { proof-of-overlay-hello = prev.hello; @@ -24,4 +24,53 @@ lib.recurseIntoAttrs { machine.succeed("hello | figlet >/dev/console") ''; }); + + testBuildFailure = lib.recurseIntoAttrs { + happy = runCommand "testBuildFailure-happy" { + failed = testers.testBuildFailure (runCommand "fail" {} '' + echo ok-ish >$out + echo failing though + echo also stderr 1>&2 + exit 3 + ''); + } '' + grep -F 'failing though' $failed/testBuildFailure.log + grep -F 'also stderr' $failed/testBuildFailure.log + grep -F 'ok-ish' $failed/result + [[ 3 = $(cat $failed/testBuildFailure.exit) ]] + touch $out + ''; + + helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" { + failed = testers.testBuildFailure (testers.testBuildFailure hello); + + # Add hello itself as a prerequisite, so we don't try to run this test if + # there's an actual failure in hello. + inherit hello; + } '' + echo "Checking $failed/testBuildFailure.log" + grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log + [[ 1 = $(cat $failed/testBuildFailure.exit) ]] + touch $out + ''; + + multiOutput = runCommand "testBuildFailure-multiOutput" { + failed = testers.testBuildFailure (runCommand "fail" { + # dev will be the default output + outputs = ["dev" "doc" "out"]; + } '' + echo i am failing + exit 1 + ''); + } '' + grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null + [[ 1 = $(cat $failed/testBuildFailure.exit) ]] + + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + + touch $out + ''; + }; + }