From 1e4eb2df3fe59c4ef9a20830a18ae2771b8bfa45 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sat, 28 Mar 2026 21:35:49 +0100 Subject: [PATCH] nixos: completely remove systemd-udev-settle.service This service is essentially a random sleep() call that old software that is unable to properly handle modern hardware initialisation being asynchronous can use to *mostly* get by. Over the years is has caused so many issues in NixOS, like introducing long delays in the boot process[1], complete deadlocks[2] and even failures when reloading services or activating a new configuration[3]. systemd has been discouraging its use since 2011 (15 years ago!), then it officially deprecated it in 2018 and since 2020 it started to show on every boot a huge wall of text calling out all the offending services that still use it. Around 2021 we managed to fix around 15 NixOS modules[4] that were relying on systemd-udev-settle and practically ZFS[5] was the only remaining one. However, since then, people have actually started to bring it back with new services[6][7][8]. This is not acceptable. So, to prevent any more (lazy) uses of systemd-udev-settle, we stop providing the systemd-udev-settle.service unit entirely. For existing modules that unfortunately still need it, we replace it with the command `udevadm settle --timeout=180`, which is all that the service does. Hopefully this will also increase the awareness that it's bad and something to be fixed. Note: I tested this change using - `nixosTests.zfs`, - `nixosTests.ifstate`, - `nixosTests.misc`, - `nixosTests.openvswitch`, - `nixosTests.predictable-interface-names` - `nixosTests.nvidia-container-toolkit` and making sure that none of the 165 packages that provide upstream units (via the `systemd.packages` option) had a dependency on systemd-udev-settle.service. [1]: https://github.com/NixOS/nixpkgs/pull/25311 [2]: https://github.com/NixOS/nixpkgs/issues/107341 [3]: https://github.com/NixOS/nixpkgs/pull/113804 [4]: https://github.com/NixOS/nixpkgs/issues/73095 [5]: https://github.com/openzfs/zfs/issues/10891 [6]: https://github.com/NixOS/nixpkgs/pull/257525#discussion_r1442702970 [7]: https://github.com/NixOS/nixpkgs/pull/460075 [8]: https://github.com/NixOS/nixpkgs/pull/284507 --- .../hardware/nvidia-container-toolkit/default.nix | 6 +++++- nixos/modules/services/hardware/udev.nix | 1 - nixos/modules/services/networking/ifstate.nix | 11 ++++++++--- nixos/modules/system/boot/systemd.nix | 2 -- nixos/modules/tasks/filesystems/zfs.nix | 15 ++++++--------- nixos/modules/virtualisation/openvswitch.nix | 1 - nixos/tests/misc.nix | 3 +-- nixos/tests/predictable-interface-names.nix | 3 ++- 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix index 475a5122fd4e..1a1365c5d420 100644 --- a/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix +++ b/nixos/modules/services/hardware/nvidia-container-toolkit/default.nix @@ -320,7 +320,6 @@ systemd.services.nvidia-container-toolkit-cdi-generator = { description = "Container Device Interface (CDI) for Nvidia generator"; - after = [ "systemd-udev-settle.service" ]; requiredBy = lib.mkMerge [ (lib.mkIf config.virtualisation.docker.enable [ "docker.service" ]) (lib.mkIf config.virtualisation.podman.enable [ "podman.service" ]) @@ -329,6 +328,11 @@ serviceConfig = { RuntimeDirectory = "cdi"; RemainAfterExit = true; + # We wait for the udev events queue to empty in the *hope* that the + # devices needed here become available. This is terribly broken and + # essentially no better than a random sleep(). See PR #452645 for + # an attempt to fix this issue. + ExecStartPre = "-${lib.getExe' pkgs.systemd "udevadm"} settle --timeout=180"; ExecStart = let script = pkgs.callPackage ./cdi-generate.nix { diff --git a/nixos/modules/services/hardware/udev.nix b/nixos/modules/services/hardware/udev.nix index 6e779df94e77..1299a5115a5f 100644 --- a/nixos/modules/services/hardware/udev.nix +++ b/nixos/modules/services/hardware/udev.nix @@ -455,7 +455,6 @@ in "systemd-udevd-control.socket" "systemd-udevd-kernel.socket" "systemd-udevd.service" - "systemd-udev-settle.service" "systemd-udev-trigger.service" ]; boot.initrd.systemd.storePaths = [ diff --git a/nixos/modules/services/networking/ifstate.nix b/nixos/modules/services/networking/ifstate.nix index 051322fc1ce5..ef44463ba9cf 100644 --- a/nixos/modules/services/networking/ifstate.nix +++ b/nixos/modules/services/networking/ifstate.nix @@ -69,7 +69,6 @@ let # https://github.com/systemd/systemd/blob/main/units/systemd-networkd.service.in commonServiceConfig = { after = [ - "systemd-udev-settle.service" "network-pre.target" "systemd-sysusers.service" "systemd-sysctl.service" @@ -88,6 +87,12 @@ let "network.target" ]; + # We wait for the udev events queue to empty in the *hope* that the + # devices needed here become available. This is terribly broken and + # essentially no better than a random sleep(). + # FIXME: use .device units dependecies instead. + serviceConfig.ExecStartPre = "-${lib.getExe' pkgs.systemd "udevadm"} settle --timeout=180"; + unitConfig = { # Avoid default dependencies like "basic.target", which prevents ifstate from starting before luks is unlocked. DefaultDependencies = "no"; @@ -173,7 +178,7 @@ in etc."ifstate/ifstate.yaml".source = settingsFormat.generate "ifstate.yaml" cfg.settings cfg.package; }; - systemd.services.ifstate = commonServiceConfig // { + systemd.services.ifstate = lib.recursiveUpdate commonServiceConfig { description = "IfState"; wantedBy = [ @@ -263,7 +268,7 @@ in "remote-fs.target" ]; - services.ifstate-initrd = commonServiceConfig // { + services.ifstate-initrd = lib.recursiveUpdate commonServiceConfig { description = "IfState initrd"; wantedBy = [ diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix index 56b87de1f394..99d7f13af5e9 100644 --- a/nixos/modules/system/boot/systemd.nix +++ b/nixos/modules/system/boot/systemd.nix @@ -65,7 +65,6 @@ let "systemd-udevd-control.socket" "systemd-udevd-kernel.socket" "systemd-udevd.service" - "systemd-udev-settle.service" ] ++ (optional (!config.boot.isContainer) "systemd-udev-trigger.service") ++ [ @@ -799,7 +798,6 @@ in systemd.services.systemd-random-seed.restartIfChanged = false; systemd.services.systemd-remount-fs.restartIfChanged = false; systemd.services.systemd-update-utmp.restartIfChanged = false; - systemd.services.systemd-udev-settle.restartIfChanged = false; # Causes long delays in nixos-rebuild systemd.targets.local-fs.unitConfig.X-StopOnReconfiguration = true; systemd.targets.remote-fs.unitConfig.X-StopOnReconfiguration = true; systemd.services.systemd-importd.environment = proxy_env; diff --git a/nixos/modules/tasks/filesystems/zfs.nix b/nixos/modules/tasks/filesystems/zfs.nix index a4f27ed840a9..48f85a913779 100644 --- a/nixos/modules/tasks/filesystems/zfs.nix +++ b/nixos/modules/tasks/filesystems/zfs.nix @@ -158,16 +158,8 @@ let }: lib.nameValuePair "zfs-import-${pool}" { description = "Import ZFS pool \"${pool}\""; - # We wait for systemd-udev-settle to ensure devices are available, - # but don't *require* it, because mounts shouldn't be killed if it's stopped. - # In the future, hopefully someone will complete this: - # https://github.com/zfsonlinux/zfs/pull/4943 - wants = [ - "systemd-udev-settle.service" - ] - ++ lib.optional (config.boot.initrd.clevis.useTang) "network-online.target"; + wants = lib.optional (config.boot.initrd.clevis.useTang) "network-online.target"; after = [ - "systemd-udev-settle.service" "systemd-modules-load.service" "systemd-ask-password-console.service" ] @@ -189,6 +181,11 @@ let serviceConfig = { Type = "oneshot"; RemainAfterExit = true; + # We wait for the udev events queue to empty in the *hope* that the + # devices needed by the pool become available. This is terribly broken + # and essentially no better than a random sleep(), but we can't do any + # better, see upstream issue https://github.com/openzfs/zfs/issues/10891 + ExecStartPre = "${lib.getExe' pkgs.systemd "udevadm"} settle --timeout=180"; }; environment.ZFS_FORCE = lib.optionalString force "-f"; script = diff --git a/nixos/modules/virtualisation/openvswitch.nix b/nixos/modules/virtualisation/openvswitch.nix index 5a231fb5888c..4e75d1677955 100644 --- a/nixos/modules/virtualisation/openvswitch.nix +++ b/nixos/modules/virtualisation/openvswitch.nix @@ -67,7 +67,6 @@ in systemd.services.ovsdb = { description = "Open_vSwitch Database Server"; wantedBy = [ "multi-user.target" ]; - after = [ "systemd-udev-settle.service" ]; path = [ cfg.package ]; restartTriggers = [ db diff --git a/nixos/tests/misc.nix b/nixos/tests/misc.nix index 3e296d8d428c..12c0275b3a23 100644 --- a/nixos/tests/misc.nix +++ b/nixos/tests/misc.nix @@ -106,8 +106,7 @@ in assert "machine" == machine.succeed("hostname -s").strip() with subtest("whether systemd-udevd automatically loads modules for our hardware"): - machine.succeed("systemctl start systemd-udev-settle.service") - machine.wait_for_unit("systemd-udev-settle.service") + machine.succeed("udevadm settle --timeout=180") assert "mousedev" in machine.succeed("lsmod") with subtest("whether systemd-tmpfiles-clean works"): diff --git a/nixos/tests/predictable-interface-names.nix b/nixos/tests/predictable-interface-names.nix index 0bf2afce7a39..6282c433bae7 100644 --- a/nixos/tests/predictable-interface-names.nix +++ b/nixos/tests/predictable-interface-names.nix @@ -43,9 +43,10 @@ pkgs.lib.listToAttrs ( meta = { }; nodes.machine = - { lib, ... }: + { pkgs, lib, ... }: let script = '' + ${lib.getExe' pkgs.systemd "udevadm"} settle --timeout=180 ip link if ${lib.optionalString predictable "!"} ip link show eth0; then echo Success