diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5cc43cd071cf..f87972292a44 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -329,8 +329,16 @@ in }; castopod = runTest ./castopod.nix; centrifugo = runTest ./centrifugo.nix; - ceph-multi-node-deprecated-filestore = - runTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-multi-node-deprecated-filestore.nix; + ceph-multi-node-bluestore = runTestOn [ "aarch64-linux" "x86_64-linux" ] ( + import ./ceph-multi-node-bluestore.nix { } + ); + ceph-multi-node-bluestore-cephfs = runTestOn [ "aarch64-linux" "x86_64-linux" ] ( + import ./ceph-multi-node-bluestore.nix { withCephfs = true; } + ); + ceph-multi-node-deprecated-filestore = runTestOn [ + "aarch64-linux" + "x86_64-linux" + ] ./ceph-multi-node-deprecated-filestore.nix; ceph-single-node-bluestore = runTestOn [ "aarch64-linux" "x86_64-linux" diff --git a/nixos/tests/ceph-multi-node-bluestore.nix b/nixos/tests/ceph-multi-node-bluestore.nix new file mode 100644 index 000000000000..96fefca78230 --- /dev/null +++ b/nixos/tests/ceph-multi-node-bluestore.nix @@ -0,0 +1,602 @@ +# Multi-node Ceph cluster test using BlueStore OSDs. +{ + withCephfs ? false, +}: +{ lib, ... }: +let + # Development knobs: + # * `defaultTimeout` caps how long every `waitUntilSucceeds` waits before it + # gives up. Lower it for faster feedback while iterating on the test. + # * When `investigateFailures` is true, a `waitUntilSucceeds` that times out + # does NOT fail the test; instead it dumps each machine's full (multi-boot) + # journal and `/var/log/ceph` into a `test-failure-investigation/` directory + # inside the test's working/output directory and then lets the test succeed, + # so the collected logs end up in the build's store path for convenient + # reading. Keep this `false` for real CI runs. + defaultTimeout = 60; + investigateFailures = false; + + cfg = { + clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03"; + monA = { + name = "a"; + ip = "192.168.1.1"; + }; + osd0 = { + name = "0"; + ip = "192.168.1.2"; + key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg=="; + uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9"; + }; + osd1 = { + name = "1"; + ip = "192.168.1.3"; + key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ=="; + uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5"; + }; + osd2 = { + name = "2"; + ip = "192.168.1.4"; + key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w=="; + uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f"; + }; + # Client that mounts CephFS using the in-kernel client. + kclient = { + ip = "192.168.1.5"; + }; + # Client that mounts CephFS using the `ceph-fuse` client. + fuseclient = { + ip = "192.168.1.6"; + }; + }; + generateCephConfig = + { daemonConfig }: + { + enable = true; + global = { + fsid = cfg.clusterId; + monHost = cfg.monA.ip; + monInitialMembers = cfg.monA.name; + }; + } + // daemonConfig; + + generateHost = + { cephConfig, networkConfig }: + { pkgs, ... }: + { + virtualisation = { + # A single raw block device per machine, consumed directly by BlueStore + # as `/dev/vdb`. Because BlueStore owns the raw device (there is no + # filesystem to mount), the OSD's on-disk state survives a hard + # crash/reboot and the OSD comes back automatically. + emptyDiskImages = [ 20480 ]; + vlans = [ 1 ]; + }; + + networking = networkConfig; + + # TODO: Why do we need any of these? Shouldn't Ceph work independent of `systemPackages`? Only `sudo` and `ceph` are used in our own test code. + environment.systemPackages = with pkgs; [ + bash + sudo + ceph + netcat + ]; + + services.ceph = cephConfig; + + # Restart limits are unsuitable for daemons that must recover from + # arbitrary crash/network downtimes. + # Ensure all daemons have infinite restart limits. + # Otherwise the tests are flaky based on timing. + systemd.services = + let + daemonUnits = + lib.concatMap + ( + daemonType: + lib.optionals (cephConfig.${daemonType}.enable or false) ( + map (daemon: "ceph-${daemonType}-${daemon}") cephConfig.${daemonType}.daemons + ) + ) + [ + "mon" + "mgr" + "osd" + "mds" + ]; + in + lib.genAttrs daemonUnits (_: { + serviceConfig.Restart = lib.mkForce "always"; + serviceConfig.RestartSec = lib.mkForce "1"; + unitConfig.StartLimitIntervalSec = lib.mkForce 0; # Ensure Restart=always is always honoured (no start limit) + }); + }; + + networkMonA = { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [ + { + address = cfg.monA.ip; + prefixLength = 24; + } + ]; + firewall = { + allowedTCPPorts = [ + 6789 + 3300 + ]; + allowedTCPPortRanges = [ + { + from = 6800; + to = 7300; + } + ]; + }; + }; + cephConfigMonA = generateCephConfig { + daemonConfig = { + mon = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + mgr = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + } + # The MDS daemon (which provides CephFS) is only configured in the CephFS + # variant of this test. + // lib.optionalAttrs withCephfs { + mds = { + enable = true; + daemons = [ cfg.monA.name ]; + }; + }; + }; + + networkOsd = osd: { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [ + { + address = osd.ip; + prefixLength = 24; + } + ]; + firewall = { + allowedTCPPortRanges = [ + { + from = 6800; + to = 7300; + } + ]; + }; + }; + + cephConfigOsd = + osd: + generateCephConfig { + daemonConfig = { + osd = { + enable = true; + daemons = [ osd.name ]; + }; + }; + }; + + # The CephFS clients only need the ceph client tooling. They do not run any + # ceph daemon, so they only get a minimal ceph config pointing at the mon. + # The in-kernel ceph filesystem module is part of the default kernel and is + # autoloaded by `mount -t ceph`, so no extra modules are needed. + networkClient = client: { + dhcpcd.enable = false; + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [ + { + address = client.ip; + prefixLength = 24; + } + ]; + }; + + cephConfigClient = generateCephConfig { daemonConfig = { }; }; + + generateClientHost = + { networkConfig }: + { pkgs, ... }: + { + virtualisation = { + vlans = [ 1 ]; + }; + + networking = networkConfig; + + environment.systemPackages = with pkgs; [ + ceph + ]; + + services.ceph = cephConfigClient; + }; + + # Python prelude that must run before any test code. + # It replaces every machine's `wait_until_succeeds()` with a wrapper that + # implements the "development knobs" above. + helperScript = '' + import os + import shutil + + DEFAULT_TIMEOUT = ${toString defaultTimeout} + INVESTIGATE_FAILURES = ${if investigateFailures then "True" else "False"} + + def collect_failure_investigation(failed_machine, command): + out_dir = os.path.join(driver.out_dir, "test-failure-investigation") + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, "README.txt"), "w") as f: + f.write( + "wait_until_succeeds timed out on machine " + f"'{failed_machine.name}' running command:\n{command}\n" + ) + for m in machines: + try: + m.execute("journalctl --no-pager --boot=all > /tmp/journal.txt 2>&1; true") + m.copy_from_machine("/tmp/journal.txt", out_dir) + shutil.move( + os.path.join(out_dir, "journal.txt"), + os.path.join(out_dir, f"{m.name}-journal.txt"), + ) + except Exception as e: + m.log(f"could not collect journal: {e}") + try: + m.copy_from_machine("/var/log/ceph", out_dir) + shutil.move( + os.path.join(out_dir, "ceph"), + os.path.join(out_dir, f"{m.name}-ceph-logs"), + ) + except Exception as e: + m.log(f"could not collect /var/log/ceph: {e}") + + # Replace the test driver's `wait_until_succeeds` with our wrapper. + machine_class = machines[0].__class__ + orig_wait_until_succeeds = machine_class.wait_until_succeeds + + def patched_wait_until_succeeds(self, command: str, timeout: int = DEFAULT_TIMEOUT) -> str: + try: + return orig_wait_until_succeeds(self, command, timeout=timeout) + except Exception: + if not INVESTIGATE_FAILURES: + raise + self.log( + "wait_until_succeeds timed out; collecting logs into " + "test-failure-investigation/ and ending the test as 'passed' " + "(investigateFailures = true)" + ) + collect_failure_investigation(self, command) + os._exit(0) + + # Use `setattr` (rather than a plain attribute assignment) so the type + # checker does not treat this as implicitly shadowing the driver's + # `wait_until_succeeds`; the replacement is intentional. + setattr(machine_class, "wait_until_succeeds", patched_wait_until_succeeds) + ''; + + # Set up the cluster (mon, mgr, BlueStore OSDs) and perform a + # hard whole-cluster crash/recovery (failover) test. + # + # Based on the "manual deployment" approach from: + # https://docs.ceph.com/en/tentacle/install/manual-deployment/ + baseScript = '' + start_all() + + monA.wait_for_unit("network.target") + osd0.wait_for_unit("network.target") + osd1.wait_for_unit("network.target") + osd2.wait_for_unit("network.target") + + # Bootstrap ceph-mon daemon + monA.succeed( + "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", + "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", + "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", + "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap", + "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring", + "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/", + "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done", + "systemctl start ceph-mon-${cfg.monA.name}", + ) + monA.wait_for_unit("ceph-mon-${cfg.monA.name}") + monA.succeed("ceph mon enable-msgr2") + monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false") + + # Can't check ceph status until a mon is up + monA.succeed("ceph -s | grep 'mon: 1 daemons'") + + # Start the ceph-mgr daemon, it has no deps and hardly any setup + monA.succeed( + "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring", + "sync", # to ensure shell redirection above is durable + "systemctl start ceph-mgr-${cfg.monA.name}", + ) + monA.wait_for_unit("ceph-mgr-a") + monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'") + monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") + + # Send the admin keyring to the OSD machines. + monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared") + osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph") + osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph") + osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph") + + # Bootstrap the BlueStore OSDs. + osd0.succeed( + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}", + "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd0.name}/type", + "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}/block", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}", + 'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -', + ) + osd1.succeed( + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}", + "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd1.name}/type", + "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}/block", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}", + 'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -', + ) + osd2.succeed( + "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}", + "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type", + "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block", + "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}", + 'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -', + ) + + # We `sync` so that the config survives the forced crashes below. + osd0.succeed( + "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "sync", + "systemctl start ceph-osd-${cfg.osd0.name}", + ) + osd1.succeed( + "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "sync", + "systemctl start ceph-osd-${cfg.osd1.name}", + ) + osd2.succeed( + "ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}", + "chown -R ceph:ceph /var/lib/ceph/osd", + "sync", + "systemctl start ceph-osd-${cfg.osd2.name}", + ) + monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'") + monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") + + monA.succeed( + "ceph osd pool create multi-node-test 32 32", + "ceph osd pool ls | grep 'multi-node-test'", + + # A pool that has no application associated with it stays unhealthy in + # state POOL_APP_NOT_ENABLED. Ceph only auto-associates an application + # for pools it manages itself, such as CephFS data/metadata pools + # (application "cephfs") and the pools RGW creates (application "rgw"); see + # https://docs.ceph.com/en/tentacle/rados/operations/pools/#associating-a-pool-with-an-application + # This is a plain RADOS pool, so we have to associate an application + # with it ourselves. We use the custom application name "nixos-test". + "ceph osd pool application enable multi-node-test nixos-test", + + "ceph osd pool rename multi-node-test multi-node-other-test", + "ceph osd pool ls | grep 'multi-node-other-test'", + ) + monA.succeed("ceph osd pool set multi-node-other-test size 2") + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") + monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'") + monA.fail( + "ceph osd pool ls | grep 'multi-node-test'", + "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it", + ) + + # Shut down ceph on all machines in a very unpolite way + monA.crash() + osd0.crash() + osd1.crash() + osd2.crash() + + # Start it up + osd0.start() + osd1.start() + osd2.start() + monA.start() + + # Ensure the cluster comes back up again. + monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'") + monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'") + monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'") + monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") + + # Verify the recovery. + monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'", timeout=60) + ''; + + # For `withCephfs = true`, after the regular failover, test CephFS and mounts: + # * bring up an MDS + # * create a CephFS + # * mount it via in-kernel ("kclient") and a `ceph-fuse` ("fuseclient") + # from 2 different nodes + # * verify bidirectional visibility of file changes + # * performs another crash/recovery test (without crashing the clients) + # * verifiy the mounts survive and keep working. + cephfsScript = '' + kclient.wait_for_unit("network.target") + fuseclient.wait_for_unit("network.target") + + # Start the ceph-mds daemon (which provides CephFS), after creating + # its keyring and data dir. + monA.succeed( + "sudo -u ceph mkdir -p /var/lib/ceph/mds/ceph-${cfg.monA.name}/", + "ceph auth get-or-create mds.${cfg.monA.name} mon 'allow profile mds' mgr 'allow profile mds' osd 'allow rwx' mds 'allow' > /var/lib/ceph/mds/ceph-${cfg.monA.name}/keyring", + "chown ceph:ceph /var/lib/ceph/mds/ceph-${cfg.monA.name}/keyring", + "sync", # to ensure config survives the forced crashes below + "systemctl start ceph-mds-${cfg.monA.name}", + ) + monA.wait_for_unit("ceph-mds-${cfg.monA.name}") + + # Create a CephFS. + monA.succeed( + "ceph osd pool create cephfs-data 32 32", + "ceph osd pool create cephfs-metadata 32 32", + "ceph fs new cephfs cephfs-metadata cephfs-data", + ) + # Wait for the MDS to claim the filesystem and become active. + monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60) + + # Distribute the admin keyring (and a plain secret file for the kernel + # client) to both client machines, so that they can authenticate. + monA.succeed( + "cp /etc/ceph/ceph.client.admin.keyring /tmp/shared", + "ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > /tmp/shared/admin.secret", + ) + kclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph") + fuseclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph") + kclient.succeed("cp /tmp/shared/admin.secret /etc/ceph/admin.secret") + + # Mount CephFS on the kernel client. + # We force the messenger v2 protocol via "ms_mode=secure"; the cluster + # has msgr2 enabled (see "ceph mon enable-msgr2" above) and the legacy v1 + # protocol apparently does not reconnect reliably after the servers are restarted. + # The msgr2 monitor listens on port 3300 (instead of legacy v1 port 6789), + # so we have to point the device string at that port explicitly. + # `recover_session=clean` makes the kernel client automatically reconnect + # (discarding its stale session) after the whole cluster has been down, + # which would otherwise leave the mount blocklisted and hanging forever. + # Real CephFS use may not prefer hanging `recover_session=clean`, and + # prefer manual de-blocklisting to avoid any failed OS syscalls, + # but for this test, discarding stale sessions is good enough. + kclient.succeed("mkdir -p /mnt/cephfs") + kclient.wait_until_succeeds( + "mount -t ceph ${cfg.monA.ip}:3300:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret,ms_mode=secure,recover_session=clean" + ) + kclient.succeed("mountpoint /mnt/cephfs") + + # Mount CephFS on the FUSE client using ceph-fuse. + fuseclient.succeed("mkdir -p /mnt/cephfs") + fuseclient.wait_until_succeeds( + "ceph-fuse --id admin -m ${cfg.monA.ip}:6789 /mnt/cephfs" + ) + fuseclient.succeed("mountpoint /mnt/cephfs") + + # Both clients mount the same CephFS, so files written by one must be + # visible to the other. Verify this in both directions. + + # Kernel client writes, FUSE client reads. + kclient.succeed("echo 'written by kclient' > /mnt/cephfs/from-kclient") + fuseclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-kclient)\" = 'written by kclient'" + ) + + # FUSE client writes, kernel client reads. + fuseclient.succeed("echo 'written by fuseclient' > /mnt/cephfs/from-fuseclient") + kclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-fuseclient)\" = 'written by fuseclient'" + ) + + # Crash test with CephFS. + # We deliberately do not crash the CephFS clients here: Their mounts must + # survive the (temporary) outage of the ceph servers and resume working + # once the cluster is healthy again. + monA.crash() + osd0.crash() + osd1.crash() + osd2.crash() + + # Start it up + osd0.start() + osd1.start() + osd2.start() + monA.start() + + # Ensure the cluster comes back up again. + # See the note above on why this uses `wait_until_succeeds`. + monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'") + monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'") + monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'") + monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") + + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'", timeout=60) + + # Ensure the MDS/CephFS comes back up again, too. + monA.wait_for_unit("ceph-mds-${cfg.monA.name}") + monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60) + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") + + # The clients kept running across the outage, so their CephFS mounts + # should still be present and should reconnect automatically. + kclient.succeed("mountpoint /mnt/cephfs") + fuseclient.succeed("mountpoint /mnt/cephfs") + + # The files written before the crash must still have the correct content. + kclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-fuseclient)\" = 'written by fuseclient'" + ) + fuseclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-kclient)\" = 'written by kclient'" + ) + + # Ensure the mounts are still writable after recovery, in both directions. + kclient.succeed("echo 'written by kclient after recovery' > /mnt/cephfs/from-kclient-2") + fuseclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-kclient-2)\" = 'written by kclient after recovery'" + ) + fuseclient.succeed("echo 'written by fuseclient after recovery' > /mnt/cephfs/from-fuseclient-2") + kclient.wait_until_succeeds( + "test \"$(cat /mnt/cephfs/from-fuseclient-2)\" = 'written by fuseclient after recovery'" + ) + ''; + +in +{ + name = "basic-multi-node-ceph-cluster-bluestore" + lib.optionalString withCephfs "-cephfs"; + meta = { + maintainers = with lib.maintainers; [ + nh2 + benaryorg + ]; + }; + + nodes = { + monA = generateHost { + cephConfig = cephConfigMonA; + networkConfig = networkMonA; + }; + osd0 = generateHost { + cephConfig = cephConfigOsd cfg.osd0; + networkConfig = networkOsd cfg.osd0; + }; + osd1 = generateHost { + cephConfig = cephConfigOsd cfg.osd1; + networkConfig = networkOsd cfg.osd1; + }; + osd2 = generateHost { + cephConfig = cephConfigOsd cfg.osd2; + networkConfig = networkOsd cfg.osd2; + }; + } + # CephFS client machines are only needed when testing CephFS. + // lib.optionalAttrs withCephfs { + kclient = generateClientHost { + networkConfig = networkClient cfg.kclient; + }; + fuseclient = generateClientHost { + networkConfig = networkClient cfg.fuseclient; + }; + }; + + testScript = + { ... }: + '' + ${helperScript} + ${baseScript} + ${lib.optionalString withCephfs cephfsScript} + ''; +} diff --git a/pkgs/by-name/ce/ceph/ceph.nix b/pkgs/by-name/ce/ceph/ceph.nix index 4d9c3c9744bc..1b1c208d693d 100644 --- a/pkgs/by-name/ce/ceph/ceph.nix +++ b/pkgs/by-name/ce/ceph/ceph.nix @@ -393,6 +393,8 @@ stdenv.mkDerivation { pythonEnv = ceph-python-env; tests = { inherit (nixosTests) + ceph-multi-node-bluestore + ceph-multi-node-bluestore-cephfs ceph-multi-node-deprecated-filestore ceph-single-node-bluestore ceph-single-node-bluestore-dmcrypt