mirror of
https://github.com/NixOS/nixpkgs.git
synced 2026-09-30 03:39:59 +00:00
nixos-rebuild-ng: avoid crashing when fds are closed during switch by using systemd-run --wait --verbose instead of systemd-run --pipe (#506897)
This commit is contained in:
Thiago Kenji Okada
@@ -1322,6 +1322,9 @@ in
|
||||
nixos-rebuild-target-host = runTest {
|
||||
imports = [ ./nixos-rebuild-target-host.nix ];
|
||||
};
|
||||
nixos-rebuild-target-host-interrupted = runTest {
|
||||
imports = [ ./nixos-rebuild-target-host-interrupted.nix ];
|
||||
};
|
||||
nixpkgs = pkgs.callPackage ../modules/misc/nixpkgs/test.nix { inherit evalMinimalConfig; };
|
||||
nixpkgs-config-allow-unfree =
|
||||
pkgs.callPackage ../modules/misc/nixpkgs/test-nixpkgs-config-allow-unfree.nix
|
||||
|
||||
236
nixos/tests/nixos-rebuild-target-host-interrupted.nix
Normal file
236
nixos/tests/nixos-rebuild-target-host-interrupted.nix
Normal file
@@ -0,0 +1,236 @@
|
||||
{ hostPkgs, ... }:
|
||||
|
||||
# This test recreates a remote deployment scenario where the connection
|
||||
# between deployer and target is closed during the deployment - in this
|
||||
# case because the connection goes over a 'reverse ssh' tunnel service
|
||||
# that has changes that are being deployed.
|
||||
|
||||
# This is not seamless (the deployer doesn't get to see the logs after
|
||||
# the disconnect), but is a lot better than the old behaviour, where
|
||||
# the switch was aborted and the connection never restored.
|
||||
|
||||
{
|
||||
name = "nixos-rebuild-target-host-interrupted";
|
||||
|
||||
# TODO: remove overlay from nixos/modules/profiles/installation-device.nix
|
||||
# make it a _small package instead, then remove pkgsReadOnly = false;.
|
||||
node.pkgsReadOnly = false;
|
||||
|
||||
# disabled by default. See all-tests.nix / tag(no-nix-by-default)
|
||||
defaults.nix.enable = true;
|
||||
|
||||
nodes = {
|
||||
deployer =
|
||||
{
|
||||
nodes,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
../modules/profiles/installation-device.nix
|
||||
];
|
||||
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ ];
|
||||
hashed-mirrors = null;
|
||||
connect-timeout = 1;
|
||||
};
|
||||
|
||||
system.includeBuildDependencies = true;
|
||||
|
||||
virtualisation = {
|
||||
cores = 2;
|
||||
memorySize = 3072;
|
||||
};
|
||||
|
||||
services.openssh.enable = true;
|
||||
users.users.root.openssh.authorizedKeys.keys = [ nodes.target.system.build.publicKey ];
|
||||
|
||||
system.build.privateKey = snakeOilPrivateKey;
|
||||
system.build.publicKey = snakeOilPublicKey;
|
||||
system.switch.enable = true;
|
||||
|
||||
services.getty.autologinUser = lib.mkForce "root";
|
||||
};
|
||||
|
||||
target =
|
||||
{
|
||||
nodes,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
|
||||
targetConfig = {
|
||||
documentation.enable = false;
|
||||
services.openssh.enable = true;
|
||||
system.build.privateKey = snakeOilPrivateKey;
|
||||
system.build.publicKey = snakeOilPublicKey;
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
|
||||
users.users.alice.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
|
||||
users.users.bob.openssh.authorizedKeys.keys = [ nodes.deployer.system.build.publicKey ];
|
||||
|
||||
users.users.alice.extraGroups = [ "wheel" ];
|
||||
users.users.bob.extraGroups = [ "wheel" ];
|
||||
|
||||
# Disable sudo for root to ensure sudo isn't called without `--sudo`
|
||||
security.sudo.extraRules = lib.mkForce [
|
||||
{
|
||||
groups = [ "wheel" ];
|
||||
commands = [ { command = "ALL"; } ];
|
||||
}
|
||||
{
|
||||
users = [ "alice" ];
|
||||
commands = [
|
||||
{
|
||||
command = "ALL";
|
||||
options = [ "NOPASSWD" ];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
nix.settings.trusted-users = [ "@wheel" ];
|
||||
|
||||
environment.etc."autossh-identity.key" = {
|
||||
source = nodes.target.system.build.privateKey;
|
||||
mode = "0600";
|
||||
};
|
||||
|
||||
services.autossh-ng.sessions.will-be-interrupted-by-rebuild = {
|
||||
user = "root";
|
||||
destination = "deployer";
|
||||
extraArguments = "-R2222:localhost:22 -i/etc/autossh-identity.key";
|
||||
hostKeyChecking = false;
|
||||
};
|
||||
# Faster retry to avoid slow test
|
||||
systemd.services.autossh-ng-will-be-interrupted-by-rebuild.serviceConfig.RestartSec =
|
||||
lib.mkForce "1s";
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = [ ./common/user-account.nix ];
|
||||
|
||||
config = lib.mkMerge [
|
||||
targetConfig
|
||||
{
|
||||
system.build = {
|
||||
inherit targetConfig;
|
||||
};
|
||||
system.switch.enable = true;
|
||||
|
||||
networking.hostName = "target";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
let
|
||||
sshConfig = builtins.toFile "ssh.conf" ''
|
||||
UserKnownHostsFile=/dev/null
|
||||
StrictHostKeyChecking=no
|
||||
'';
|
||||
|
||||
targetConfigJSON = hostPkgs.writeText "target-configuration.json" (
|
||||
builtins.toJSON nodes.target.system.build.targetConfig
|
||||
);
|
||||
|
||||
targetNetworkJSON = hostPkgs.writeText "target-network.json" (
|
||||
builtins.toJSON nodes.target.system.build.networkConfig
|
||||
);
|
||||
|
||||
configFile =
|
||||
hostname:
|
||||
hostPkgs.writeText "configuration.nix" # nix
|
||||
''
|
||||
{ lib, pkgs, modulesPath, ... }: {
|
||||
imports = [
|
||||
(modulesPath + "/virtualisation/qemu-vm.nix")
|
||||
(modulesPath + "/virtualisation/guest-networking-options.nix")
|
||||
(modulesPath + "/testing/test-instrumentation.nix")
|
||||
(modulesPath + "/../tests/common/user-account.nix")
|
||||
(lib.modules.importJSON ./target-configuration.json)
|
||||
(lib.modules.importJSON ./target-network.json)
|
||||
./hardware-configuration.nix
|
||||
];
|
||||
|
||||
boot.loader.grub = {
|
||||
enable = true;
|
||||
device = "/dev/vda";
|
||||
forceInstall = true;
|
||||
};
|
||||
|
||||
# We're changing the '-E' parameter to the new hostname here,
|
||||
# not because we care about the logs, but because we want to
|
||||
# force the scenario where the connection is broken during the
|
||||
# deployment (because the autossh-ng service is stopped and
|
||||
# started):
|
||||
services.autossh-ng.sessions.will-be-interrupted-by-rebuild.extraArguments = "-R2222:localhost:22 -i/etc/autossh-identity.key -E ${hostname}";
|
||||
|
||||
# this will be asserted to validate the switch happened:
|
||||
networking.hostName = "${hostname}";
|
||||
}
|
||||
'';
|
||||
in
|
||||
# python
|
||||
''
|
||||
start_all()
|
||||
target.wait_for_open_port(22)
|
||||
|
||||
deployer.wait_until_succeeds("ping -c1 target")
|
||||
deployer.succeed("install -Dm 600 ${nodes.deployer.system.build.privateKey} ~root/.ssh/id_ecdsa")
|
||||
deployer.succeed("install ${sshConfig} ~root/.ssh/config")
|
||||
|
||||
target.succeed("nixos-generate-config")
|
||||
deployer.succeed("scp alice@target:/etc/nixos/hardware-configuration.nix /root/hardware-configuration.nix")
|
||||
target.wait_for_unit("autossh-ng-will-be-interrupted-by-rebuild.service")
|
||||
|
||||
deployer.copy_from_host("${configFile "config-1-deployed"}", "/root/configuration-1.nix")
|
||||
deployer.copy_from_host("${configFile "config-2-deployed"}", "/root/configuration-2.nix")
|
||||
deployer.copy_from_host("${targetNetworkJSON}", "/root/target-network.json")
|
||||
deployer.copy_from_host("${targetConfigJSON}", "/root/target-configuration.json")
|
||||
|
||||
with subtest("Deploy to alice@target via reverse ssh"):
|
||||
deployer.wait_for_unit("multi-user.target")
|
||||
# Uses TTY/send_chars instead of deployer.succeed to set NIX_SSHOPTS
|
||||
deployer.send_chars("NIX_SSHOPTS=\"-p 2222\" nixos-rebuild switch -I nixos-config=/root/configuration-1.nix --target-host alice@localhost --sudo\n")
|
||||
|
||||
# the connection breaks, but the 'switch' should now continue in the background:
|
||||
deployer.wait_until_tty_matches("1", "error: while running command with remote sudo")
|
||||
|
||||
def deployed(last_try: bool) -> bool:
|
||||
target_hostname = deployer.succeed("ssh alice@target cat /etc/hostname", timeout=20).rstrip()
|
||||
if last_try:
|
||||
print(f"Still seeing hostname {target_hostname}")
|
||||
return target_hostname == "config-1-deployed"
|
||||
retry(deployed)
|
||||
|
||||
with subtest("Deploy to bob@target via reverse ssh with password-based sudo"):
|
||||
deployer.wait_for_unit("multi-user.target")
|
||||
# Uses TTY/send_chars instead of deployer.succeed to set NIX_SSHOPTS and for ask-sudo-password
|
||||
deployer.send_chars("""NIX_SSHOPTS="-p 2222" nixos-rebuild switch -I nixos-config=/root/configuration-2.nix --target-host bob@localhost --ask-sudo-password; printf '%s\\n' "$?" > /tmp/bob-rebuild-status\n""")
|
||||
deployer.wait_until_tty_matches("1", "password for bob")
|
||||
deployer.send_chars("${nodes.target.users.users.bob.password}\n")
|
||||
|
||||
# the connection breaks, but the 'switch' should now continue in the background:
|
||||
deployer.wait_for_file("/tmp/bob-rebuild-status")
|
||||
status = deployer.succeed("cat /tmp/bob-rebuild-status").strip()
|
||||
assert status != "0", "Expected the interrupted SSH deployment to report failure"
|
||||
|
||||
def deployed(last_try: bool) -> bool:
|
||||
target_hostname = deployer.succeed("ssh alice@target cat /etc/hostname", timeout=20).rstrip()
|
||||
if last_try:
|
||||
print(f"Still seeing hostname {target_hostname}")
|
||||
return target_hostname == "config-2-deployed"
|
||||
retry(deployed)
|
||||
'';
|
||||
}
|
||||
@@ -46,7 +46,9 @@ SWITCH_TO_CONFIGURATION_CMD_PREFIX: Final = [
|
||||
"NIXOS_NO_CHECK",
|
||||
"--collect",
|
||||
"--no-ask-password",
|
||||
"--pipe",
|
||||
"--wait",
|
||||
"--verbose",
|
||||
"--output=cat",
|
||||
"--quiet",
|
||||
"--service-type=exec",
|
||||
"--unit=nixos-rebuild-switch-to-configuration",
|
||||
|
||||
Reference in New Issue
Block a user