From 5f5734db9d73fc3f4caa4be18995bba9b4c58c84 Mon Sep 17 00:00:00 2001 From: Michael Schneider Date: Wed, 15 Apr 2026 12:14:00 +0100 Subject: [PATCH] nixos/test-driver: add option to force kvm use --- nixos/doc/manual/redirects.json | 3 ++ nixos/lib/qemu-common.nix | 23 +++++---- .../lib/test-driver/src/test_driver/driver.py | 20 +++++++- .../src/test_driver/machine/__init__.py | 26 +++++++++- nixos/lib/testing/driver.nix | 10 ++++ nixos/lib/testing/nodes.nix | 4 +- nixos/modules/virtualisation/qemu-vm.nix | 47 ++++++++++++++++++- 7 files changed, 120 insertions(+), 13 deletions(-) diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index 71a137b0fa33..4dbb255682ed 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -2246,6 +2246,9 @@ "test-opt-passthru": [ "index.html#test-opt-passthru" ], + "test-opt-qemu.forceAccel": [ + "index.html#test-opt-qemu.forceAccel" + ], "test-opt-qemu.package": [ "index.html#test-opt-qemu.package" ], diff --git a/nixos/lib/qemu-common.nix b/nixos/lib/qemu-common.nix index a43b624cdbce..9ff482b83f3a 100644 --- a/nixos/lib/qemu-common.nix +++ b/nixos/lib/qemu-common.nix @@ -24,30 +24,37 @@ rec { else throw "Unknown QEMU serial device for system '${stdenv.hostPlatform.system}'"; - qemuBinary = - qemuPkg: + qemuBinary = qemuPkg: qemuBinaryWith { inherit qemuPkg; }; + + qemuBinaryWith = + { + qemuPkg, + forceAccel ? false, + }: let hostStdenv = qemuPkg.stdenv; hostSystem = hostStdenv.system; guestSystem = stdenv.hostPlatform.system; + accel = accelName: if forceAccel then accelName else "${accelName}:tcg"; + linuxHostGuestMatrix = { - x86_64-linux = "${qemuPkg}/bin/qemu-system-x86_64 -machine accel=kvm:tcg -cpu max"; - armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -machine virt,accel=kvm:tcg -cpu max"; - aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=max,accel=kvm:tcg -cpu max"; + x86_64-linux = "${qemuPkg}/bin/qemu-system-x86_64 -machine accel=${accel "kvm"} -cpu max"; + armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -machine virt,accel=${accel "kvm"} -cpu max"; + aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=max,accel=${accel "kvm"} -cpu max"; powerpc64le-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv"; powerpc64-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv"; riscv32-linux = "${qemuPkg}/bin/qemu-system-riscv32 -machine virt"; riscv64-linux = "${qemuPkg}/bin/qemu-system-riscv64 -machine virt"; - x86_64-darwin = "${qemuPkg}/bin/qemu-system-x86_64 -machine accel=kvm:tcg -cpu max"; + x86_64-darwin = "${qemuPkg}/bin/qemu-system-x86_64 -machine accel=${accel "kvm"} -cpu max"; }; otherHostGuestMatrix = { aarch64-darwin = { - aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=2,accel=hvf:tcg -cpu max"; + aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=2,accel=${accel "hvf"} -cpu max"; inherit (otherHostGuestMatrix.x86_64-darwin) x86_64-linux; }; x86_64-darwin = { - x86_64-linux = "${qemuPkg}/bin/qemu-system-x86_64 -machine type=q35,accel=hvf:tcg -cpu max"; + x86_64-linux = "${qemuPkg}/bin/qemu-system-x86_64 -machine type=q35,accel=${accel "hvf"} -cpu max"; }; }; diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index abaa2cc71955..714a11905306 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -336,10 +336,22 @@ class Driver: def start_all(self) -> None: """Start all machines""" with self.logger.nested("start all VMs"): + errors: list[tuple[str, BaseException]] = [] + + def start_machine(machine: BaseMachine) -> None: + try: + machine.start() + except Exception as e: + errors.append((machine.name, e)) + threads = [] for machine in self.machines: # Create a thread for each machine's start method - t = threading.Thread(target=machine.start, name=f"start-{machine.name}") + t = threading.Thread( + target=start_machine, + args=(machine,), + name=f"start-{machine.name}", + ) threads.append(t) t.start() @@ -347,6 +359,12 @@ class Driver: for t in threads: t.join() + if errors: + messages = [f"{name}: {e}" for name, e in errors] + raise MachineError( + "Failed to start the following machines:\n" + "\n".join(messages) + ) + def join_all(self) -> None: """Wait for all machines to shut down""" with self.logger.nested("wait for all VMs to finish"): diff --git a/nixos/lib/test-driver/src/test_driver/machine/__init__.py b/nixos/lib/test-driver/src/test_driver/machine/__init__.py index 7edb72b76145..62c10d425f86 100644 --- a/nixos/lib/test-driver/src/test_driver/machine/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/machine/__init__.py @@ -212,6 +212,7 @@ class QemuStartCommand: ), stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True, cwd=state_dir, env=self.build_environment(state_dir, shared_dir), @@ -1227,8 +1228,29 @@ class QemuMachine(BaseMachine): self.shell_path, allow_reboot, ) - self.monitor, _ = monitor_socket.accept() - self.shell, _ = shell_socket.accept() + + def accept_or_fail(sock: socket.socket, name: str) -> socket.socket: + """Accept a connection on a socket, polling the status to check + if the QEMU process is still alive. Without this, socket.accept() + would block forever if QEMU exits before connecting. + """ + assert self.process + while True: + readable, _, _ = select.select([sock], [], [], 1.0) + if readable: + conn, _ = sock.accept() + return conn + rc = self.process.poll() + if rc is not None: + output = "" + if self.process.stdout: + output = self.process.stdout.read().decode(errors="ignore") + raise MachineError( + f"QEMU process exited with code {rc} before connecting to {name} socket.\n{output}" + ) + + self.monitor = accept_or_fail(monitor_socket, "monitor") + self.shell = accept_or_fail(shell_socket, "shell") self.qmp_client = QMPSession.from_path(self.qmp_path) # Store last serial console lines for use diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix index d5638718dda7..e9808907b2c5 100644 --- a/nixos/lib/testing/driver.nix +++ b/nixos/lib/testing/driver.nix @@ -159,6 +159,16 @@ in defaultText = "hostPkgs.qemu_test"; }; + qemu.forceAccel = mkOption { + description = '' + Whether to force the use of hardware-accelerated virtualisation. + When enabled, QEMU will not fall back to the slower software emulation + (TCG) and will instead error out if the accelerator is not available. + ''; + type = types.bool; + default = false; + }; + globalTimeout = mkOption { description = '' A global timeout for the complete test, expressed in seconds. diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix index 7384dec68895..e1a148297e32 100644 --- a/nixos/lib/testing/nodes.nix +++ b/nixos/lib/testing/nodes.nix @@ -71,7 +71,9 @@ let config.nodeDefaults { key = "base-qemu"; - virtualisation.qemu.package = testModuleArgs.config.qemu.package; + virtualisation.qemu = { + inherit (testModuleArgs.config.qemu) package forceAccel; + }; virtualisation.host.pkgs = hostPkgs; } testModuleArgs.config.extraBaseNodeModules diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 33bd759a1e3f..efc7921da00b 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -306,8 +306,42 @@ let (builtins.concatStringsSep "") ]} + ${lib.optionalString cfg.qemu.forceAccel ( + if hostPkgs.stdenv.hostPlatform.isLinux then + '' + # Check for hardware-accelerated virtualisation support (KVM) + if [ ! -e /dev/kvm ]; then + echo "forceAccel is enabled but /dev/kvm does not exist." >&2 + echo "Hardware-accelerated virtualisation (KVM) is not available on this system." >&2 + exit 1 + elif [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + echo "forceAccel is enabled but /dev/kvm is not accessible (permission denied)." >&2 + echo "Check that the nix build user is in the 'kvm' group or that /dev/kvm has the correct permissions." >&2 + exit 1 + fi + '' + else if hostPkgs.stdenv.hostPlatform.isDarwin then + '' + # Check for hardware-accelerated virtualisation support (HVF) + if ! sysctl -n kern.hv_support 2>/dev/null | grep -q 1; then + echo "forceAccel is enabled but Hypervisor.framework is not available on this system." >&2 + exit 1 + fi + '' + else + '' + echo "forceAccel is enabled but no known accelerator is available for this platform." >&2 + exit 1 + '' + )} + # Start QEMU. - exec ${qemu-common.qemuBinary qemu} \ + exec ${ + qemu-common.qemuBinaryWith { + qemuPkg = qemu; + forceAccel = cfg.qemu.forceAccel; + } + } \ -name ${config.system.name} \ -m ${toString config.virtualisation.memorySize} \ -smp ${toString config.virtualisation.cores} \ @@ -728,6 +762,17 @@ in description = "QEMU package to use."; }; + forceAccel = mkOption { + type = types.bool; + default = false; + description = '' + Whether to force the use of hardware-accelerated virtualisation. + When enabled, QEMU will not fall back to the slower software + emulation (TCG) and will instead error out if the accelerator is not + available. + ''; + }; + options = mkOption { type = types.listOf types.str; default = [ ];