Merge branch 'staging-nixos' into staging-next

nixos/lib/test-driver/src/test_driver/machine/__init__.py

@@ -335,7 +335,7 @@ class BaseMachine(ABC):
         ...
 
     @abstractmethod
-    def wait_for_shutdown(self) -> None:
+    def wait_for_shutdown(self, timeout: dt.timedelta | None = None) -> None:
         """Wait for the machine to power off. This does *not* initiate a shutdown;
         that's usually done via `shutdown()`.
         """
@@ -1061,7 +1061,7 @@ class QemuMachine(BaseMachine):
                 break
             self.send_console(char.decode())
 
-    def wait_for_shutdown(self) -> None:
+    def wait_for_shutdown(self, timeout: dt.timedelta | None = None) -> None:
         """
         Wait for the VM to power off. This does *not* initiate a shutdown;
         that's usually done via `shutdown()`.
@@ -1072,7 +1072,9 @@ class QemuMachine(BaseMachine):
         with self.nested("waiting for the VM to power off"):
             sys.stdout.flush()
             assert self.process
-            self.process.wait()
+            self.process.wait(
+                timeout=timeout.total_seconds() if timeout is not None else None
+            )
 
             self.pid = None
             self.booted = False
@@ -1903,7 +1905,7 @@ class NspawnMachine(BaseMachine):
         self.systemctl("poweroff")
         self.wait_for_shutdown()
 
-    def wait_for_shutdown(self) -> None:
+    def wait_for_shutdown(self, timeout: dt.timedelta | None = None) -> None:
         """
         Wait for the container to power off. This does *not* initiate a shutdown;
         that's usually done via `shutdown()`.
@@ -1912,7 +1914,9 @@ class NspawnMachine(BaseMachine):
             return
 
         with self.nested("waiting for the container to power off"):
-            self.process.wait()
+            self.process.wait(
+                timeout=timeout.total_seconds() if timeout is not None else None
+            )
             self.process = None
 
 

nixos/tests/all-tests.nix

@@ -1322,6 +1322,9 @@ in
   nixos-rebuild-target-host = runTest {
     imports = [ ./nixos-rebuild-target-host.nix ];
   };
+  nixos-rebuild-target-host-interrupted = runTest {
+    imports = [ ./nixos-rebuild-target-host-interrupted.nix ];
+  };
   nixpkgs = pkgs.callPackage ../modules/misc/nixpkgs/test.nix { inherit evalMinimalConfig; };
   nixpkgs-config-allow-unfree =
     pkgs.callPackage ../modules/misc/nixpkgs/test-nixpkgs-config-allow-unfree.nix

nixos/tests/nixos-rebuild-target-host-interrupted.nix

@@ -0,0 +1,236 @@
+{ hostPkgs, ... }:
+
+# This test recreates a remote deployment scenario where the connection
+# between deployer and target is closed during the deployment - in this
+# case because the connection goes over a 'reverse ssh' tunnel service
+# that has changes that are being deployed.
+
+# This is not seamless (the deployer doesn't get to see the logs after
+# the disconnect), but is a lot better than the old behaviour, where
+# the switch was aborted and the connection never restored.
+
+{
+  name = "nixos-rebuild-target-host-interrupted";
+
+  # TODO: remove overlay from  nixos/modules/profiles/installation-device.nix
+  #        make it a _small package instead, then remove pkgsReadOnly = false;.
+  node.pkgsReadOnly = false;
+
+  # disabled by default. See all-tests.nix / tag(no-nix-by-default)
+  defaults.nix.enable = true;
+
+  nodes = {
+    deployer =
+      {
+        nodes,
+        lib,
+        pkgs,
+        ...
+      }:
+      let
+        inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
+      in
+      {
+        imports = [
+          ../modules/profiles/installation-device.nix
+        ];
+
+        nix.settings = {
+          substituters = lib.mkForce [ ];
+          hashed-mirrors = null;
+          connect-timeout = 1;
+        };
+
+        system.includeBuildDependencies = true;
+
+        virtualisation = {
+          cores = 2;
+          memorySize = 3072;
+        };
+
+        services.openssh.enable = true;
+        users.users.root.openssh.authorizedKeys.keys = [ nodes.target.system.build.publicKey ];
+
+        system.build.privateKey = snakeOilPrivateKey;
+        system.build.publicKey = snakeOilPublicKey;
+        system.switch.enable = true;
+
+        services.getty.autologinUser = lib.mkForce "root";
+      };
+
+    target =
+      {
+        nodes,
+        lib,
+        pkgs,
+        ...
+      }:
+      let
+        inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
+        targetConfig = {
+          documentation.enable = false;
+          services.openssh.enable = true;
+          system.build.privateKey = snakeOilPrivateKey;
+          system.build.publicKey = snakeOilPublicKey;
+
+          users.users.root.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
+          users.users.alice.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
+          users.users.bob.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
+
+          users.users.alice.extraGroups = [ "wheel" ];
+          users.users.bob.extraGroups = [ "wheel" ];
+
+          # Disable sudo for root to ensure sudo isn't called without `--sudo`
+          security.sudo.extraRules = lib.mkForce [
+            {
+              groups = [ "wheel" ];
+              commands = [ { command = "ALL"; } ];
+            }
+            {
+              users = [ "alice" ];
+              commands = [
+                {
+                  command = "ALL";
+                  options = [ "NOPASSWD" ];
+                }
+              ];
+            }
+          ];
+
+          nix.settings.trusted-users = [ "@wheel" ];
+
+          environment.etc."autossh-identity.key" = {
+            source = nodes.target.system.build.privateKey;
+            mode = "0600";
+          };
+
+          services.autossh-ng.sessions.will-be-interrupted-by-rebuild = {
+            user = "root";
+            destination = "deployer";
+            extraArguments = "-R2222:localhost:22 -i/etc/autossh-identity.key";
+            hostKeyChecking = false;
+          };
+          # Faster retry to avoid slow test
+          systemd.services.autossh-ng-will-be-interrupted-by-rebuild.serviceConfig.RestartSec =
+            lib.mkForce "1s";
+        };
+      in
+      {
+        imports = [ ./common/user-account.nix ];
+
+        config = lib.mkMerge [
+          targetConfig
+          {
+            system.build = {
+              inherit targetConfig;
+            };
+            system.switch.enable = true;
+
+            networking.hostName = "target";
+          }
+        ];
+      };
+  };
+
+  testScript =
+    { nodes, ... }:
+    let
+      sshConfig = builtins.toFile "ssh.conf" ''
+        UserKnownHostsFile=/dev/null
+        StrictHostKeyChecking=no
+      '';
+
+      targetConfigJSON = hostPkgs.writeText "target-configuration.json" (
+        builtins.toJSON nodes.target.system.build.targetConfig
+      );
+
+      targetNetworkJSON = hostPkgs.writeText "target-network.json" (
+        builtins.toJSON nodes.target.system.build.networkConfig
+      );
+
+      configFile =
+        hostname:
+        hostPkgs.writeText "configuration.nix" # nix
+          ''
+            { lib, pkgs, modulesPath, ... }: {
+              imports = [
+                (modulesPath + "/virtualisation/qemu-vm.nix")
+                (modulesPath + "/virtualisation/guest-networking-options.nix")
+                (modulesPath + "/testing/test-instrumentation.nix")
+                (modulesPath + "/../tests/common/user-account.nix")
+                (lib.modules.importJSON ./target-configuration.json)
+                (lib.modules.importJSON ./target-network.json)
+                ./hardware-configuration.nix
+              ];
+
+              boot.loader.grub = {
+                enable = true;
+                device = "/dev/vda";
+                forceInstall = true;
+              };
+
+              # We're changing the '-E' parameter to the new hostname here,
+              # not because we care about the logs, but because we want to
+              # force the scenario where the connection is broken during the
+              # deployment (because the autossh-ng service is stopped and
+              # started):
+              services.autossh-ng.sessions.will-be-interrupted-by-rebuild.extraArguments = "-R2222:localhost:22 -i/etc/autossh-identity.key -E ${hostname}";
+
+              # this will be asserted to validate the switch happened:
+              networking.hostName = "${hostname}";
+            }
+          '';
+    in
+    # python
+    ''
+      start_all()
+      target.wait_for_open_port(22)
+
+      deployer.wait_until_succeeds("ping -c1 target")
+      deployer.succeed("install -Dm 600 ${nodes.deployer.system.build.privateKey} ~root/.ssh/id_ecdsa")
+      deployer.succeed("install ${sshConfig} ~root/.ssh/config")
+
+      target.succeed("nixos-generate-config")
+      deployer.succeed("scp alice@target:/etc/nixos/hardware-configuration.nix /root/hardware-configuration.nix")
+      target.wait_for_unit("autossh-ng-will-be-interrupted-by-rebuild.service")
+
+      deployer.copy_from_host("${configFile "config-1-deployed"}", "/root/configuration-1.nix")
+      deployer.copy_from_host("${configFile "config-2-deployed"}", "/root/configuration-2.nix")
+      deployer.copy_from_host("${targetNetworkJSON}", "/root/target-network.json")
+      deployer.copy_from_host("${targetConfigJSON}", "/root/target-configuration.json")
+
+      with subtest("Deploy to alice@target via reverse ssh"):
+        deployer.wait_for_unit("multi-user.target")
+        # Uses TTY/send_chars instead of deployer.succeed to set NIX_SSHOPTS
+        deployer.send_chars("NIX_SSHOPTS=\"-p 2222\" nixos-rebuild switch -I nixos-config=/root/configuration-1.nix --target-host alice@localhost --sudo\n")
+
+        # the connection breaks, but the 'switch' should now continue in the background:
+        deployer.wait_until_tty_matches("1", "error: while running command with remote sudo")
+
+        def deployed(last_try: bool) -> bool:
+            target_hostname = deployer.succeed("ssh alice@target cat /etc/hostname", timeout=20).rstrip()
+            if last_try:
+                print(f"Still seeing hostname {target_hostname}")
+            return target_hostname == "config-1-deployed"
+        retry(deployed)
+
+      with subtest("Deploy to bob@target via reverse ssh with password-based sudo"):
+        deployer.wait_for_unit("multi-user.target")
+        # Uses TTY/send_chars instead of deployer.succeed to set NIX_SSHOPTS and for ask-sudo-password
+        deployer.send_chars("""NIX_SSHOPTS="-p 2222" nixos-rebuild switch -I nixos-config=/root/configuration-2.nix --target-host bob@localhost --ask-sudo-password; printf '%s\\n' "$?" > /tmp/bob-rebuild-status\n""")
+        deployer.wait_until_tty_matches("1", "password for bob")
+        deployer.send_chars("${nodes.target.users.users.bob.password}\n")
+
+        # the connection breaks, but the 'switch' should now continue in the background:
+        deployer.wait_for_file("/tmp/bob-rebuild-status")
+        status = deployer.succeed("cat /tmp/bob-rebuild-status").strip()
+        assert status != "0", "Expected the interrupted SSH deployment to report failure"
+
+        def deployed(last_try: bool) -> bool:
+            target_hostname = deployer.succeed("ssh alice@target cat /etc/hostname", timeout=20).rstrip()
+            if last_try:
+                print(f"Still seeing hostname {target_hostname}")
+            return target_hostname == "config-2-deployed"
+        retry(deployed)
+    '';
+}

nixos/tests/systemd-shutdown.nix

@@ -13,24 +13,32 @@ in
 
   nodes.machine = {
     imports = [ ../modules/profiles/minimal.nix ];
+    systemd.shutdown.pre-exitrd = pkgs.writeShellScript "pre-exitrd" ''
+      echo pre-exitrd > /run/initramfs/test.txt
+    '';
     systemd.shutdownRamfs.contents."/etc/systemd/system-shutdown/shutdown-message".source =
       pkgs.writeShellScript "shutdown-message" ''
-        echo "${msg}" > /dev/kmsg
+        if test -e /test.txt; then
+          # Test should only pass if both scripts run.
+          echo "${msg}" > /dev/kmsg
+        fi
       '';
     boot.initrd.systemd.enable = systemdStage1;
   };
 
   testScript = ''
+    import datetime as dt
+
     # Check that 'generate-shutdown-ramfs.service' is started
     # automatically and that 'systemd-shutdown' runs our script.
     machine.wait_for_unit("multi-user.target")
     # .shutdown() would wait for the machine to power off
     machine.execute("systemctl poweroff", check_return=False)
     # Message printed by systemd-shutdown
-    machine.wait_for_console_text("Unmounting '/oldroot'")
-    machine.wait_for_console_text("${msg}")
+    machine.wait_for_console_text("Unmounting '/oldroot'", timeout=dt.timedelta(seconds=60))
+    machine.wait_for_console_text("${msg}", timeout=dt.timedelta(seconds=5))
     # Don't try to sync filesystems
-    machine.wait_for_shutdown()
+    machine.wait_for_shutdown(timeout=dt.timedelta(seconds=5))
 
     # In a separate boot, start 'generate-shutdown-ramfs.service'
     # manually in order to check the permissions on '/run/initramfs'.

pkgs/by-name/bl/blender-oneapi/package.nix

@@ -0,0 +1,3 @@
+{ blender }:
+
+blender.override { oneapiSupport = true; }

pkgs/by-name/bl/blender/package.nix

@@ -86,6 +86,12 @@
   waylandSupport ? stdenv.hostPlatform.isLinux,
   zlib,
   zstd,
+  level-zero,
+  intel-compute-runtime,
+  intel-llvm,
+  intel-graphics-compiler,
+  oneapiSupport ? false,
+  opencl-headers,
 }:
 
 let
@@ -121,6 +127,9 @@ stdenv'.mkDerivation (finalAttrs: {
   pname = "blender";
   version = "5.2.2";
 
+  strictDeps = true;
+  __structuredAttrs = true;
+
   src = fetchzip {
     name = "source";
     url = "https://download.blender.org/source/blender-${finalAttrs.version}.tar.xz";
@@ -158,6 +167,10 @@ stdenv'.mkDerivation (finalAttrs: {
   + (lib.optionalString rocmSupport ''
     substituteInPlace extern/hipew/src/hipew.c --replace-fail '"/opt/rocm/hip/lib/libamdhip64.so.${lib.versions.major rocmPackages.clr.version}"' '"${rocmPackages.clr}/lib/libamdhip64.so"'
     substituteInPlace extern/hipew/src/hipew.c --replace-fail '"opt/rocm/hip/bin"' '"${rocmPackages.clr}/bin"'
+  '')
+  + (lib.optionalString oneapiSupport ''
+    substituteInPlace intern/cycles/kernel/device/oneapi/CMakeLists.txt \
+      --replace-fail ''\'''${cycles_kernel_runtime_lib_target_path}' '"''${CMAKE_INSTALL_LIBDIR}"'
   '');
 
   env.NIX_CFLAGS_COMPILE = "-I${python3}/include/${python3.libPrefix}";
@@ -177,7 +190,8 @@ stdenv'.mkDerivation (finalAttrs: {
     (lib.cmakeBool "WITH_CPU_CHECK" false)
     (lib.cmakeBool "WITH_CYCLES_CUDA_BINARIES" cudaSupport)
     (lib.cmakeBool "WITH_CYCLES_DEVICE_HIP" rocmSupport)
-    (lib.cmakeBool "WITH_CYCLES_DEVICE_ONEAPI" false)
+    (lib.cmakeBool "WITH_CYCLES_DEVICE_ONEAPI" oneapiSupport)
+    (lib.cmakeBool "WITH_CYCLES_ONEAPI_BINARIES" oneapiSupport)
     (lib.cmakeBool "WITH_CYCLES_DEVICE_OPTIX" cudaSupport)
     (lib.cmakeBool "WITH_CYCLES_EMBREE" embreeSupport)
     (lib.cmakeBool "WITH_CYCLES_OSL" true)
@@ -205,6 +219,13 @@ stdenv'.mkDerivation (finalAttrs: {
     (lib.cmakeFeature "OPTIX_ROOT_DIR" "${optix}")
     (lib.cmakeBool "WITH_CYCLES_CUDA_BINARIES" true)
   ]
+  ++ lib.optionals oneapiSupport [
+    (lib.cmakeFeature "SYCL_ROOT_DIR" "${intel-llvm}")
+    (lib.cmakeFeature "LEVEL_ZERO_ROOT_DIR" "${level-zero}")
+    (lib.cmakeFeature "OCLOC_INSTALL_DIR" "${intel-compute-runtime}")
+    (lib.cmakeFeature "IGC_INSTALL_DIR" "${intel-graphics-compiler}")
+    (lib.cmakeFeature "SYCL_CPP_FLAGS" "--verbose")
+  ]
   ++ lib.optionals rocmSupport [
     (lib.cmakeBool "WITH_CYCLES_DEVICE_HIPRT" false)
     (lib.cmakeBool "WITH_CYCLES_HIP_BINARIES" true)
@@ -236,17 +257,15 @@ stdenv'.mkDerivation (finalAttrs: {
 
   nativeBuildInputs = [
     cmake
-    llvmPackages.llvm.dev
     makeWrapper
+    pkg-config
     python3Packages.wrapPython
+    python3
   ]
+  ++ lib.optional oneapiSupport addDriverRunpath
   ++ lib.optionals cudaSupport [
     addDriverRunpath
     cudaPackages.cuda_nvcc
-  ]
-  ++ lib.optionals waylandSupport [
-    pkg-config
-    wayland-scanner
   ];
 
   buildInputs = [
@@ -283,13 +302,18 @@ stdenv'.mkDerivation (finalAttrs: {
     openxr-loader
     potrace
     pugixml
-    python3
     python3Packages.materialx
     python3Packages.openshadinglanguage
     rubberband
     zlib
     zstd
   ]
+  ++ lib.optionals oneapiSupport [
+    intel-compute-runtime
+    intel-llvm
+    opencl-headers
+  ]
+  ++ lib.optional (!oneapiSupport) llvmPackages.llvm
   ++ lib.optional embreeSupport embree
   ++ lib.optional rocmSupport rocmPackages.clr
   ++ lib.optional openImageDenoiseSupport (openimagedenoise.override { inherit cudaSupport; })
@@ -325,6 +349,7 @@ stdenv'.mkDerivation (finalAttrs: {
     libxkbcommon
     wayland
     wayland-protocols
+    wayland-scanner
   ]
   ++ lib.optional jackaudioSupport libjack2
   ++ lib.optional spaceNavSupport libspnav
@@ -370,10 +395,10 @@ stdenv'.mkDerivation (finalAttrs: {
         --add-flags '--python-use-system-env'
     '';
 
-  # Set RUNPATH so that libcuda and libnvrtc in /run/opengl-driver(-32)/lib can be
+  # Set RUNPATH so that libs in /run/opengl-driver(-32)/lib can be
   # found. See the explanation in libglvnd.
   postFixup =
-    lib.optionalString cudaSupport ''
+    lib.optionalString (cudaSupport || oneapiSupport) ''
       for program in $out/bin/blender $out/bin/.blender-wrapped; do
         addDriverRunpath "$program"
       done

pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py

@@ -46,7 +46,9 @@ SWITCH_TO_CONFIGURATION_CMD_PREFIX: Final = [
     "NIXOS_NO_CHECK",
     "--collect",
     "--no-ask-password",
-    "--pipe",
+    "--wait",
+    "--verbose",
+    "--output=cat",
     "--quiet",
     "--service-type=exec",
     "--unit=nixos-rebuild-switch-to-configuration",

pkgs/by-name/un/unifont/package.nix

@@ -15,13 +15,13 @@ let
 in
 stdenv.mkDerivation (finalAttrs: {
   pname = "unifont";
-  version = "17.0.05";
+  version = "18.0.01";
 
   strictDeps = true;
 
   src = fetchurl {
     url = "mirror://gnu/unifont/unifont-${finalAttrs.version}/unifont-${finalAttrs.version}.tar.gz";
-    hash = "sha256-8ofP+ybiJyOqNuZoSGmw8/87+4IsSwEAi9hHkR7BtjE=";
+    hash = "sha256-6rYIR6rDTIdodlzsx4Ifr1DeJjYYe0Urm1+lChKwC8M=";
   };
 
   postPatch = ''