diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 8e5e3fc66417..bd58f29f7925 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -340,22 +340,10 @@ in
   ceph-multi-node-bluestore-cephfs = runTestOn [ "aarch64-linux" "x86_64-linux" ] (
     import ./ceph-multi-node-bluestore.nix { withCephfs = true; }
   );
-  ceph-multi-node-deprecated-filestore = runTestOn [
-    "aarch64-linux"
-    "x86_64-linux"
-  ] ./ceph-multi-node-deprecated-filestore.nix;
   ceph-single-node-bluestore = runTestOn [
     "aarch64-linux"
     "x86_64-linux"
   ] ./ceph-single-node-bluestore.nix;
-  ceph-single-node-bluestore-dmcrypt = runTestOn [
-    "aarch64-linux"
-    "x86_64-linux"
-  ] ./ceph-single-node-bluestore-dmcrypt.nix;
-  ceph-single-node-deprecated-filestore = runTestOn [
-    "aarch64-linux"
-    "x86_64-linux"
-  ] ./ceph-single-node-deprecated-filestore.nix;
   certmgr = import ./certmgr.nix { inherit pkgs runTest; };
   cfssl = runTestOn [ "aarch64-linux" "x86_64-linux" ] ./cfssl.nix;
   cgit = runTest ./cgit.nix;
diff --git a/nixos/tests/ceph-multi-node-bluestore.nix b/nixos/tests/ceph-multi-node-bluestore.nix
index 96fefca78230..9f61df41ee35 100644
--- a/nixos/tests/ceph-multi-node-bluestore.nix
+++ b/nixos/tests/ceph-multi-node-bluestore.nix
@@ -25,19 +25,16 @@ let
     osd0 = {
       name = "0";
       ip = "192.168.1.2";
-      key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
       uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
     };
     osd1 = {
       name = "1";
       ip = "192.168.1.3";
-      key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
       uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
     };
     osd2 = {
       name = "2";
       ip = "192.168.1.4";
-      key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
       uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
     };
     # Client that mounts CephFS using the in-kernel client.
@@ -58,6 +55,14 @@ let
         monHost = cfg.monA.ip;
         monInitialMembers = cfg.monA.name;
       };
+      extraConfig = {
+        log_to_syslog = "false";
+        log_to_file = "false";
+        log_to_stderr = "true";
+        debug_rocksdb = "1/5";
+        debug_mgr = "1/5";
+        mon_host = "v2:${cfg.monA.ip}:3300 v1:${cfg.monA.ip}:6789";
+      };
     }
     // daemonConfig;
 
@@ -81,6 +86,7 @@ let
         bash
         sudo
         ceph
+        cryptsetup
         netcat
       ];
 
@@ -145,6 +151,11 @@ let
         enable = true;
         daemons = [ cfg.monA.name ];
       };
+      # TODO: move this to a separate machine
+      rgw = {
+        enable = true;
+        daemons = [ cfg.monA.name ];
+      };
     }
     # The MDS daemon (which provides CephFS) is only configured in the CephFS
     # variant of this test.
@@ -209,6 +220,11 @@ let
         vlans = [ 1 ];
       };
 
+      # Ceph 20.2.4 introduced the aes256k cipher for authentication.
+      # Linux started supporting these in kernel version 7.0.
+      # Remove this line at the earliest convenience (i.e. when tests are run by 7.0 or higher by default).
+      boot.kernelPackages = pkgs.linuxPackages_latest;
+
       networking = networkConfig;
 
       environment.systemPackages = with pkgs; [
@@ -285,6 +301,8 @@ let
   # Based on the "manual deployment" approach from:
   # https://docs.ceph.com/en/tentacle/install/manual-deployment/
   baseScript = ''
+    import json
+
     start_all()
 
     monA.wait_for_unit("network.target")
@@ -297,14 +315,15 @@ let
         "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
         "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
         "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-        "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
+        # Creating the mon with v2 (and a legacy v1) address right away removes the need for running `enable-msgr2` later on.
+        # It is also makes the test more consistent by fixing the address to a known value instead of letting it derive the address.
+        "monmaptool --create --addv ${cfg.monA.name} '[v2:${cfg.monA.ip}:3300,v1:${cfg.monA.ip}:6789]' --auth-allowed-ciphers aes256k --auth-preferred-cipher aes256k --auth-service-cipher aes256k --fsid ${cfg.clusterId} /tmp/monmap",
         "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
         "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
         "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
         "systemctl start ceph-mon-${cfg.monA.name}",
     )
     monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
-    monA.succeed("ceph mon enable-msgr2")
     monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
 
     # Can't check ceph status until a mon is up
@@ -320,59 +339,63 @@ let
     monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
     monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
 
-    # Send the admin keyring to the OSD machines.
-    monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared")
-    osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-    osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-    osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
+    # Send the bootstrap-osd keyring to the OSD machines.
+    monA.succeed("ceph auth get client.bootstrap-osd -o /etc/ceph/ceph.client.bootstrap-osd.keyring")
+    monA.succeed("cp /etc/ceph/ceph.client.bootstrap-osd.keyring /tmp/shared")
 
     # Bootstrap the BlueStore OSDs.
-    osd0.succeed(
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-        "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd0.name}/type",
-        "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}/block",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
-        'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
-    )
-    osd1.succeed(
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-        "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd1.name}/type",
-        "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}/block",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
-        'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
-    )
-    osd2.succeed(
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
-        "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type",
-        "ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
-        'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
-    )
+    #
+    # The steps for this are roughly the same for all OSDs:
+    # 1. get the bootstrap-osd keyring
+    # 2. prepare the osd via ceph-volume lvm, the second line contains the OSD specific configuration
+    # 3. deactivate it to unmount the tmpfs
+    # 4. activate it without a tmpfs for persistent data
+    # 5. sync, so the osd has at least one consistent state saved
+    # 6. start it
 
-    # We `sync` so that the config survives the forced crashes below.
+    # osd.0: plain
     osd0.succeed(
-        "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
-        "chown -R ceph:ceph /var/lib/ceph/osd",
+        "mkdir -p /var/lib/ceph/bootstrap-osd",
+        "cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
+        "ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-id ${cfg.osd0.name} --osd-fsid ${cfg.osd0.uuid} "
+          "--data /dev/vdb",
+        "ceph-volume lvm deactivate ${cfg.osd0.name} ${cfg.osd0.uuid}",
+        "ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd0.name} ${cfg.osd0.uuid}",
         "sync",
         "systemctl start ceph-osd-${cfg.osd0.name}",
     )
+    # osd.1: plain
     osd1.succeed(
-        "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
-        "chown -R ceph:ceph /var/lib/ceph/osd",
+        "mkdir -p /var/lib/ceph/bootstrap-osd",
+        "cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
+        "ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-id ${cfg.osd1.name} --osd-fsid ${cfg.osd1.uuid} "
+          "--data /dev/vdb --dmcrypt",
+        "ceph-volume lvm deactivate ${cfg.osd1.name} ${cfg.osd1.uuid}",
+        "ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
         "sync",
         "systemctl start ceph-osd-${cfg.osd1.name}",
     )
+    # osd.2: plain
     osd2.succeed(
-        "ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
-        "chown -R ceph:ceph /var/lib/ceph/osd",
+        "mkdir -p /var/lib/ceph/bootstrap-osd",
+        "cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
+        "ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-fsid ${cfg.osd2.uuid} --osd-id ${cfg.osd2.name} "
+          "--data /dev/vdb",
+        "ceph-volume lvm deactivate ${cfg.osd2.name} ${cfg.osd2.uuid}",
+        "ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd2.name} ${cfg.osd2.uuid}",
         "sync",
         "systemctl start ceph-osd-${cfg.osd2.name}",
     )
+
+
     monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
     monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
     monA.succeed(
+        # Autoscaling will cause PGs to be peering, causing the tests to become flakey.
+        "ceph osd pool set noautoscale",
+
         "ceph osd pool create multi-node-test 32 32",
         "ceph osd pool ls | grep 'multi-node-test'",
 
@@ -389,6 +412,7 @@ let
         "ceph osd pool ls | grep 'multi-node-other-test'",
     )
     monA.succeed("ceph osd pool set multi-node-other-test size 2")
+    # TODO: actually write to the pool using rados directly
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
     monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'")
     monA.fail(
@@ -396,23 +420,100 @@ let
         "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",
     )
 
+    # Bootstrap RGW
+    monA.succeed(
+        "sudo -u ceph mkdir -p /var/lib/ceph/radosgw/ceph-${cfg.monA.name}",
+        "ceph auth get-or-create client.${cfg.monA.name} osd 'allow rwx' mon 'allow rw' > /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
+        "chown ceph:ceph /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
+        "systemctl start ceph-rgw-${cfg.monA.name}",
+    )
+    monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
+    monA.wait_for_open_port(7480)
+
+    # Enable the dashboard and recheck health
+    monA.succeed(
+        "ceph mgr module enable dashboard",
+        "ceph config set mgr mgr/dashboard/ssl false",
+        # default is 8080 but it's better to be explicit
+        "ceph config set mgr mgr/dashboard/server_port 8080",
+    )
+
+    # The dashboard does not listen on localhost:
+    # `server_addr` defaults to the wildcard address, but the dashboard module
+    # resolves that to the active mgr's own IP and binds only to it,
+    # so loopback is never bound.
+    # See https://github.com/ceph/ceph/blob/v20.2.2/src/pybind/mgr/dashboard/module.py#L213-L214
+    # Therefore address the dashboard via the mgr's IP instead of localhost.
+    dashboard = "http://${cfg.monA.ip}:8080"
+
+    monA.wait_for_open_port(8080, addr="${cfg.monA.ip}")
+    monA.wait_until_succeeds(f"curl -s --fail {dashboard}")
+    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
+
+    # Initialize dashboard creds.
+    # In a the query below, we test the Dashboard's `/api/rgw/daemon`,
+    # which needs that the dashboard can talk to RGW.
+    # `set-rgw-credentials` needs a running RGW daemon.
+    monA.succeed(
+        "echo 'foo bar baz qux' > /tmp/dashboard_pw",
+        "ceph dashboard ac-user-create admin -i /tmp/dashboard_pw administrator",
+        "ceph dashboard set-rgw-credentials",
+        "sync",
+    )
+
+    # Get dashboard auth token
+    auth_payload = json.dumps({"username": "admin", "password": "foo bar baz qux"})
+    auth_response = json.loads(monA.succeed(
+        f"curl --fail -s -X POST -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -d '{auth_payload}' {dashboard}/api/auth",
+    ))
+    token = auth_response["token"]
+
+    # Check cluster health via dashboard API
+    health = json.loads(monA.succeed(
+        f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' {dashboard}/api/health/minimal",
+    ))
+    assert health["health"]["status"] == "HEALTH_OK"
+
+    # List daemons via REST API.
+    # This also requires a running RGW daemon, as it asserts on the first one.
+    rgw_daemons = json.loads(monA.succeed(
+        f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' {dashboard}/api/rgw/daemon",
+    ))
+    assert rgw_daemons[0]["id"] == "${cfg.monA.name}"
+
     # Shut down ceph on all machines in a very unpolite way
     monA.crash()
     osd0.crash()
     osd1.crash()
     osd2.crash()
 
-    # Start it up
+    # Start the mon first and mark the OSDs as down.
+    # Since the heartbeats are pretty high by default, the OSDs would otherwise be marked as up still.
+    # However we do not want to lower the heartbeats since this might cause flakey tests.
+    monA.start()
+    monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
+    monA.wait_until_succeeds("ceph osd down all")
+    # Then start the OSDs as normal.
     osd0.start()
     osd1.start()
     osd2.start()
-    monA.start()
+    # Ensure they are all up.
+    osd0.wait_for_unit("network.target")
+    osd1.wait_for_unit("network.target")
+    osd2.wait_for_unit("network.target")
 
-    # Ensure the cluster comes back up again.
+    # FIXME: dmcrypt OSDs currently do not work out of the box.
+    # For a potential long-term fix see: https://github.com/NixOS/nixpkgs/pull/512912#discussion_r3140295546
+    osd1.succeed(
+        "ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
+        "systemctl start ceph-osd-${cfg.osd1.name}",
+    )
+
+    # Test the cluster state thoroughly.
     monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
     monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
     monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
+    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
     # Verify the recovery.
@@ -444,45 +545,50 @@ let
 
     # Create a CephFS.
     monA.succeed(
-        "ceph osd pool create cephfs-data 32 32",
-        "ceph osd pool create cephfs-metadata 32 32",
-        "ceph fs new cephfs cephfs-metadata cephfs-data",
+        "ceph fs volume create testing",
+        "ceph osd pool set cephfs.testing.data pg_num 32",
+        "ceph osd pool set cephfs.testing.meta pg_num 32",
     )
     # Wait for the MDS to claim the filesystem and become active.
-    monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
+    monA.wait_until_succeeds("ceph fs status testing | grep -e 'active'", timeout=60)
 
-    # Distribute the admin keyring (and a plain secret file for the kernel
-    # client) to both client machines, so that they can authenticate.
+    # Create a subvolume, issue credentials, then distribute those credentials.
     monA.succeed(
-        "cp /etc/ceph/ceph.client.admin.keyring /tmp/shared",
-        "ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > /tmp/shared/admin.secret",
+        "ceph fs subvolumegroup create testing group",
+        "ceph fs subvolume create testing subvolume --group_name group",
+        "ceph fs subvolume authorize testing subvolume kclient group",
+        "ceph fs subvolume authorize testing subvolume fuseclient group",
+        "ceph auth get client.kclient -o /tmp/shared/ceph.client.kclient.keyring",
+        "ceph auth get client.fuseclient -o /tmp/shared/ceph.client.fuseclient.keyring",
     )
-    kclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-    fuseclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-    kclient.succeed("cp /tmp/shared/admin.secret /etc/ceph/admin.secret")
+    kclient.succeed("cp /tmp/shared/ceph.client.kclient.keyring /etc/ceph")
+    fuseclient.succeed("cp /tmp/shared/ceph.client.fuseclient.keyring /etc/ceph")
+
+    # Get the volume path generated by Ceph.
+    volume_path = monA.succeed("ceph fs subvolume getpath testing subvolume group | tee /dev/stderr").strip()
 
     # Mount CephFS on the kernel client.
     # We force the messenger v2 protocol via "ms_mode=secure"; the cluster
-    # has msgr2 enabled (see "ceph mon enable-msgr2" above) and the legacy v1
+    # has msgr2 enabled (the monmap is created with a v2 address above) and the legacy v1
     # protocol apparently does not reconnect reliably after the servers are restarted.
     # The msgr2 monitor listens on port 3300 (instead of legacy v1 port 6789),
     # so we have to point the device string at that port explicitly.
     # `recover_session=clean` makes the kernel client automatically reconnect
     # (discarding its stale session) after the whole cluster has been down,
-    # which would otherwise leave the mount blocklisted and hanging forever.
+    # which would otherwise leave the mount blocklisted and hanging.
     # Real CephFS use may not prefer hanging `recover_session=clean`, and
     # prefer manual de-blocklisting to avoid any failed OS syscalls,
     # but for this test, discarding stale sessions is good enough.
     kclient.succeed("mkdir -p /mnt/cephfs")
     kclient.wait_until_succeeds(
-        "mount -t ceph ${cfg.monA.ip}:3300:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret,ms_mode=secure,recover_session=clean"
+        f"mount -t ceph kclient@.testing={volume_path} /mnt/cephfs -o ms_mode=secure,recover_session=clean"
     )
     kclient.succeed("mountpoint /mnt/cephfs")
 
     # Mount CephFS on the FUSE client using ceph-fuse.
     fuseclient.succeed("mkdir -p /mnt/cephfs")
     fuseclient.wait_until_succeeds(
-        "ceph-fuse --id admin -m ${cfg.monA.ip}:6789 /mnt/cephfs"
+        f"ceph-fuse --id fuseclient -m ${cfg.monA.ip}:3300 -r {volume_path} /mnt/cephfs"
     )
     fuseclient.succeed("mountpoint /mnt/cephfs")
 
@@ -510,24 +616,40 @@ let
     osd1.crash()
     osd2.crash()
 
-    # Start it up
+    # Start the mon first and mark the OSDs as down.
+    # Since the heartbeats are pretty high by default, the OSDs would otherwise be marked as up still.
+    # However we do not want to lower the heartbeats since this might cause flakey tests.
+    monA.start()
+    monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
+    monA.wait_until_succeeds("ceph osd down all")
+    # Then start the OSDs as normal.
     osd0.start()
     osd1.start()
     osd2.start()
-    monA.start()
+    # Ensure they are all up.
+    osd0.wait_for_unit("network.target")
+    osd1.wait_for_unit("network.target")
+    osd2.wait_for_unit("network.target")
+
+    # FIXME: dmcrypt OSDs currently do not work out of the box.
+    # For a potential long-term fix see: https://github.com/NixOS/nixpkgs/pull/512912#discussion_r3140295546
+    osd1.succeed(
+        "ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
+        "systemctl start ceph-osd-${cfg.osd1.name}",
+    )
 
     # Ensure the cluster comes back up again.
     # See the note above on why this uses `wait_until_succeeds`.
     monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
     monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
     monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
+    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
 
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'", timeout=60)
 
     # Ensure the MDS/CephFS comes back up again, too.
     monA.wait_for_unit("ceph-mds-${cfg.monA.name}")
-    monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
+    monA.wait_until_succeeds("ceph fs status testing | grep -e 'active'", timeout=60)
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
     # The clients kept running across the outage, so their CephFS mounts
diff --git a/nixos/tests/ceph-multi-node-deprecated-filestore.nix b/nixos/tests/ceph-multi-node-deprecated-filestore.nix
deleted file mode 100644
index 992a74f72972..000000000000
--- a/nixos/tests/ceph-multi-node-deprecated-filestore.nix
+++ /dev/null
@@ -1,291 +0,0 @@
-# Tests the legacy FileStore OSD backend.
-{ lib, ... }:
-let
-  cfg = {
-    clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03";
-    monA = {
-      name = "a";
-      ip = "192.168.1.1";
-    };
-    osd0 = {
-      name = "0";
-      ip = "192.168.1.2";
-      key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
-      uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
-    };
-    osd1 = {
-      name = "1";
-      ip = "192.168.1.3";
-      key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
-      uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
-    };
-    osd2 = {
-      name = "2";
-      ip = "192.168.1.4";
-      key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
-      uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
-    };
-  };
-  generateCephConfig =
-    { daemonConfig }:
-    {
-      enable = true;
-      global = {
-        fsid = cfg.clusterId;
-        monHost = cfg.monA.ip;
-        monInitialMembers = cfg.monA.name;
-      };
-    }
-    // daemonConfig;
-
-  generateHost =
-    { cephConfig, networkConfig }:
-    { pkgs, ... }:
-    {
-      virtualisation = {
-        emptyDiskImages = [ 20480 ];
-        vlans = [ 1 ];
-      };
-
-      networking = networkConfig;
-
-      environment.systemPackages = with pkgs; [
-        bash
-        sudo
-        ceph
-        xfsprogs
-        netcat
-      ];
-
-      boot.kernelModules = [ "xfs" ];
-
-      services.ceph = cephConfig;
-    };
-
-  networkMonA = {
-    dhcpcd.enable = false;
-    interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
-      {
-        address = cfg.monA.ip;
-        prefixLength = 24;
-      }
-    ];
-    firewall = {
-      allowedTCPPorts = [
-        6789
-        3300
-      ];
-      allowedTCPPortRanges = [
-        {
-          from = 6800;
-          to = 7300;
-        }
-      ];
-    };
-  };
-  cephConfigMonA = generateCephConfig {
-    daemonConfig = {
-      mon = {
-        enable = true;
-        daemons = [ cfg.monA.name ];
-      };
-      mgr = {
-        enable = true;
-        daemons = [ cfg.monA.name ];
-      };
-    };
-  };
-
-  networkOsd = osd: {
-    dhcpcd.enable = false;
-    interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
-      {
-        address = osd.ip;
-        prefixLength = 24;
-      }
-    ];
-    firewall = {
-      allowedTCPPortRanges = [
-        {
-          from = 6800;
-          to = 7300;
-        }
-      ];
-    };
-  };
-
-  cephConfigOsd =
-    osd:
-    generateCephConfig {
-      daemonConfig = {
-        osd = {
-          enable = true;
-          daemons = [ osd.name ];
-        };
-      };
-    };
-
-  # Following deployment is based on the manual deployment described here:
-  # https://docs.ceph.com/docs/master/install/manual-deployment/
-  # For other ways to deploy a ceph cluster, look at the documentation at
-  # https://docs.ceph.com/docs/master/
-  testscript =
-    { ... }:
-    ''
-      start_all()
-
-      monA.wait_for_unit("network.target")
-      osd0.wait_for_unit("network.target")
-      osd1.wait_for_unit("network.target")
-      osd2.wait_for_unit("network.target")
-
-      # Bootstrap ceph-mon daemon
-      monA.succeed(
-          "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
-          "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
-          "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-          "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
-          "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
-          "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
-          "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
-          "systemctl start ceph-mon-${cfg.monA.name}",
-      )
-      monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
-      monA.succeed("ceph mon enable-msgr2")
-      monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
-
-      # Can't check ceph status until a mon is up
-      monA.succeed("ceph -s | grep 'mon: 1 daemons'")
-
-      # Start the ceph-mgr daemon, it has no deps and hardly any setup
-      monA.succeed(
-          "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
-          "sync",  # to ensure shell redirection above is durable
-          "systemctl start ceph-mgr-${cfg.monA.name}",
-      )
-      monA.wait_for_unit("ceph-mgr-a")
-      monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-      monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-
-      # Send the admin keyring to the OSD machines
-      monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared")
-      osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-      osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-      osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
-
-      # Bootstrap OSDs
-      osd0.succeed(
-          "mkfs.xfs /dev/vdb",
-          "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-          "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-          "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
-          'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
-      )
-      osd1.succeed(
-          "mkfs.xfs /dev/vdb",
-          "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-          "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-          "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
-          'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
-      )
-      osd2.succeed(
-          "mkfs.xfs /dev/vdb",
-          "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
-          "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
-          "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
-          'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
-      )
-
-      # We `sync` so that the config survives the forced crashes below.
-      osd0.succeed(
-          "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
-          "chown -R ceph:ceph /var/lib/ceph/osd",
-          "sync",
-          "systemctl start ceph-osd-${cfg.osd0.name}",
-      )
-      osd1.succeed(
-          "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
-          "chown -R ceph:ceph /var/lib/ceph/osd",
-          "sync",
-          "systemctl start ceph-osd-${cfg.osd1.name}",
-      )
-      osd2.succeed(
-          "ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
-          "chown -R ceph:ceph /var/lib/ceph/osd",
-          "sync",
-          "systemctl start ceph-osd-${cfg.osd2.name}",
-      )
-      monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-      monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-      monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-
-      monA.succeed(
-          "ceph osd pool create multi-node-test 32 32",
-          "ceph osd pool ls | grep 'multi-node-test'",
-
-          # We need to enable an application on the pool, otherwise it will
-          # stay unhealthy in state POOL_APP_NOT_ENABLED.
-          # Creating a CephFS would do this automatically, but we haven't done that here.
-          # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
-          # We use the custom application name "nixos-test" for this.
-          "ceph osd pool application enable multi-node-test nixos-test",
-
-          "ceph osd pool rename multi-node-test multi-node-other-test",
-          "ceph osd pool ls | grep 'multi-node-other-test'",
-      )
-      monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
-      monA.succeed("ceph osd pool set multi-node-other-test size 2")
-      monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-      monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
-      monA.fail(
-          "ceph osd pool ls | grep 'multi-node-test'",
-          "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",
-      )
-
-      # Shut down ceph on all machines in a very unpolite way
-      monA.crash()
-      osd0.crash()
-      osd1.crash()
-      osd2.crash()
-
-      # Start it up
-      osd0.start()
-      osd1.start()
-      osd2.start()
-      monA.start()
-
-      # Ensure the cluster comes back up again
-      monA.succeed("ceph -s | grep 'mon: 1 daemons'")
-      monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-      monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-      monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-      monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-    '';
-in
-{
-  name = "basic-multi-node-ceph-cluster-deprecated-filestore";
-  meta = with lib.maintainers; {
-    maintainers = [ lejonet ];
-  };
-
-  nodes = {
-    monA = generateHost {
-      cephConfig = cephConfigMonA;
-      networkConfig = networkMonA;
-    };
-    osd0 = generateHost {
-      cephConfig = cephConfigOsd cfg.osd0;
-      networkConfig = networkOsd cfg.osd0;
-    };
-    osd1 = generateHost {
-      cephConfig = cephConfigOsd cfg.osd1;
-      networkConfig = networkOsd cfg.osd1;
-    };
-    osd2 = generateHost {
-      cephConfig = cephConfigOsd cfg.osd2;
-      networkConfig = networkOsd cfg.osd2;
-    };
-  };
-
-  testScript = testscript;
-}
diff --git a/nixos/tests/ceph-single-node-bluestore-dmcrypt.nix b/nixos/tests/ceph-single-node-bluestore-dmcrypt.nix
deleted file mode 100644
index 8ab8cbba3a62..000000000000
--- a/nixos/tests/ceph-single-node-bluestore-dmcrypt.nix
+++ /dev/null
@@ -1,269 +0,0 @@
-{ lib, ... }:
-
-let
-  # the single node ipv6 address
-  ip = "2001:db8:ffff::";
-  # the global ceph cluster id
-  cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";
-  # the fsids of OSDs
-  osd-fsid-map = {
-    "0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";
-    "1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";
-    "2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";
-  };
-in
-{
-  name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";
-  meta.maintainers = with lib.maintainers; [
-    benaryorg
-    nh2
-  ];
-
-  nodes.ceph =
-    {
-      lib,
-      pkgs,
-      config,
-      ...
-    }:
-    {
-      # disks for bluestore
-      virtualisation.emptyDiskImages = [
-        20480
-        20480
-        20480
-      ];
-
-      # networking setup (no external connectivity required, only local IPv6)
-      networking.useDHCP = false;
-      systemd.network = {
-        enable = true;
-        wait-online.extraArgs = [
-          "-i"
-          "lo"
-        ];
-        networks = {
-          "40-loopback" = {
-            enable = true;
-            name = "lo";
-            DHCP = "no";
-            addresses = [ { Address = "${ip}/128"; } ];
-          };
-        };
-      };
-
-      # do not start the ceph target by default so we can format the disks first
-      systemd.targets.ceph.wantedBy = lib.mkForce [ ];
-
-      # add the packages to systemPackages so the testscript doesn't run into any unexpected issues
-      # this shouldn't be required on production systems which have their required packages in the unit paths only
-      # but it helps in case one needs to actually run the tooling anyway
-      environment.systemPackages = with pkgs; [
-        ceph
-        cryptsetup
-        lvm2
-      ];
-
-      services.ceph = {
-        enable = true;
-        client.enable = true;
-        extraConfig = {
-          public_addr = ip;
-          cluster_addr = ip;
-          # ipv6
-          ms_bind_ipv4 = "false";
-          ms_bind_ipv6 = "true";
-          # msgr2 settings
-          ms_cluster_mode = "secure";
-          ms_service_mode = "secure";
-          ms_client_mode = "secure";
-          ms_mon_cluster_mode = "secure";
-          ms_mon_service_mode = "secure";
-          ms_mon_client_mode = "secure";
-          # less default modules, cuts down on memory and startup time in the tests
-          mgr_initial_modules = "";
-          # distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host
-          osd_crush_chooseleaf_type = "0";
-        };
-        client.extraConfig."mon.0" = {
-          host = "ceph";
-          mon_addr = "v2:[${ip}]:3300";
-          public_addr = "v2:[${ip}]:3300";
-        };
-        global = {
-          fsid = cluster;
-          clusterNetwork = "${ip}/64";
-          publicNetwork = "${ip}/64";
-          monInitialMembers = "0";
-        };
-
-        mon = {
-          enable = true;
-          daemons = [ "0" ];
-        };
-
-        osd = {
-          enable = true;
-          daemons = builtins.attrNames osd-fsid-map;
-        };
-
-        mgr = {
-          enable = true;
-          daemons = [ "ceph" ];
-        };
-      };
-
-      systemd.services =
-        let
-          osd-name = id: "ceph-osd-${id}";
-          osd-pre-start = id: [
-            "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"
-            "${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"
-          ];
-          osd-post-stop = id: [
-            "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"
-          ];
-          map-osd = id: {
-            name = osd-name id;
-            value = {
-              serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);
-              serviceConfig.ExecStopPost = osd-post-stop id;
-              unitConfig.ConditionPathExists = lib.mkForce [ ];
-              unitConfig.StartLimitBurst = lib.mkForce 4;
-              path = with pkgs; [
-                util-linux
-                lvm2
-                cryptsetup
-              ];
-            };
-          };
-        in
-        lib.pipe config.services.ceph.osd.daemons [
-          (map map-osd)
-          builtins.listToAttrs
-        ];
-    };
-
-  testScript = ''
-    start_all()
-
-    ceph.wait_for_unit("default.target")
-
-    # Bootstrap ceph-mon daemon
-    ceph.succeed(
-        "mkdir -p /var/lib/ceph/bootstrap-osd",
-        "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
-        "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
-        "ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",
-        "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-        "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
-        "monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",
-        "mkdir -p /var/lib/ceph/mon/ceph-0",
-        "ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",
-        "chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",
-        "systemctl start ceph-mon-0.service",
-    )
-
-    ceph.wait_for_unit("ceph-mon-0.service")
-    # should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout
-    ceph.wait_for_open_port(3300, "${ip}")
-    ceph.succeed(
-        # required for HEALTH_OK
-        "ceph config set mon auth_allow_insecure_global_id_reclaim false",
-        # IPv6
-        "ceph config set global ms_bind_ipv4 false",
-        "ceph config set global ms_bind_ipv6 true",
-        # the new (secure) protocol
-        "ceph config set global ms_bind_msgr1 false",
-        "ceph config set global ms_bind_msgr2 true",
-        # just a small little thing
-        "ceph config set mon mon_compact_on_start true",
-    )
-
-    # Can't check ceph status until a mon is up
-    ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
-
-    # Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)
-    ceph.succeed(
-        # this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases
-        "ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",
-        "ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",
-        "ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",
-        "sudo ceph-volume lvm deactivate 0",
-        "sudo ceph-volume lvm deactivate 1",
-        "sudo ceph-volume lvm deactivate 2",
-        "chown -R ceph:ceph /var/lib/ceph",
-    )
-
-    # Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)
-    ceph.succeed(
-        "systemctl start ceph-osd-0.service",
-        "systemctl start ceph-osd-1.service",
-        "systemctl start ceph-osd-2.service",
-    )
-    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
-    ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-
-    # Start the ceph-mgr daemon, after copying in the keyring
-    ceph.succeed(
-        "mkdir -p /var/lib/ceph/mgr/ceph-ceph/",
-        "ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",
-        "chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",
-        "systemctl start ceph-mgr-ceph.service",
-    )
-    ceph.wait_for_unit("ceph-mgr-ceph")
-    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
-    ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
-    ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-
-    # test the actual storage
-    ceph.succeed(
-        "ceph osd pool create single-node-test 32 32",
-        "ceph osd pool ls | grep 'single-node-test'",
-
-        # We need to enable an application on the pool, otherwise it will
-        # stay unhealthy in state POOL_APP_NOT_ENABLED.
-        # Creating a CephFS would do this automatically, but we haven't done that here.
-        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
-        # We use the custom application name "nixos-test" for this.
-        "ceph osd pool application enable single-node-test nixos-test",
-
-        "ceph osd pool rename single-node-test single-node-other-test",
-        "ceph osd pool ls | grep 'single-node-other-test'",
-    )
-    ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
-    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-    ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")
-    ceph.fail(
-        # the old pool should be gone
-        "ceph osd pool ls | grep 'multi-node-test'",
-        # deleting the pool should fail without setting mon_allow_pool_delete
-        "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
-    )
-
-    # rebooting gets rid of any potential tmpfs mounts or device-mapper devices
-    ceph.shutdown()
-    ceph.start()
-    ceph.wait_for_unit("default.target")
-
-    # Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)
-    ceph.systemctl("start ceph-mon-0.service")
-    ceph.wait_for_unit("ceph-mon-0")
-    ceph.systemctl("start ceph-osd-0.service")
-    ceph.wait_for_unit("ceph-osd-0")
-    ceph.systemctl("start ceph-osd-1.service")
-    ceph.wait_for_unit("ceph-osd-1")
-    ceph.systemctl("start ceph-osd-2.service")
-    ceph.wait_for_unit("ceph-osd-2")
-    ceph.systemctl("start ceph-mgr-ceph.service")
-    ceph.wait_for_unit("ceph-mgr-ceph")
-
-    # Ensure the cluster comes back up again
-    ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
-    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
-    ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")
-    ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
-    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-  '';
-}
diff --git a/nixos/tests/ceph-single-node-bluestore.nix b/nixos/tests/ceph-single-node-bluestore.nix
index 6665a586e8a8..fac0b797683b 100644
--- a/nixos/tests/ceph-single-node-bluestore.nix
+++ b/nixos/tests/ceph-single-node-bluestore.nix
@@ -9,17 +9,14 @@ let
     };
     osd0 = {
       name = "0";
-      key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
       uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
     };
     osd1 = {
       name = "1";
-      key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
       uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
     };
     osd2 = {
       name = "2";
-      key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
       uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
     };
   };
@@ -51,6 +48,11 @@ let
         vlans = [ 1 ];
       };
 
+      # Ceph 20.2.4 introduced the aes256k cipher for authentication.
+      # Linux started supporting these in kernel version 7.0.
+      # Remove this line at the earliest convenience (i.e. when tests are run by 7.0 or higher by default).
+      boot.kernelPackages = pkgs.linuxPackages_latest;
+
       networking = networkConfig;
 
       environment.systemPackages = with pkgs; [
@@ -109,13 +111,18 @@ let
         "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
         "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
         "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-        "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
+        # Create the monmap with both a msgr2 (v2) and a legacy (v1) address.
+        # Using plain `--add` yields a v1-only monmap, which leaves the cluster
+        # in HEALTH_WARN with MON_MSGR2_NOT_ENABLED. Running `ceph mon
+        # enable-msgr2` afterwards is not enough: it rewrites the monmap (a
+        # subsequent `ceph mon dump` does show the v2 address), but the health
+        # check keeps reporting the mon as v1-only indefinitely.
+        "monmaptool --create --addv ${cfg.monA.name} '[v2:${cfg.monA.ip}:3300,v1:${cfg.monA.ip}:6789]' --fsid ${cfg.clusterId} /tmp/monmap",
         "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
         "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
         "systemctl start ceph-mon-${cfg.monA.name}",
     )
     monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
-    monA.succeed("ceph mon enable-msgr2")
     monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
 
     # Can't check ceph status until a mon is up
@@ -142,14 +149,24 @@ let
         "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
         "echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type",
         "ln -sf /dev/vdd /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
-        'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
-        'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
-        'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
+        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --gen-key",
+        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --gen-key",
+        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --gen-key",
     )
 
+    # Register the OSDs with the generated keys read back from their keyrings.
+    for osd_name, osd_uuid in [
+        ("${cfg.osd0.name}", "${cfg.osd0.uuid}"),
+        ("${cfg.osd1.name}", "${cfg.osd1.uuid}"),
+        ("${cfg.osd2.name}", "${cfg.osd2.uuid}"),
+    ]:
+        key = monA.succeed(
+            f"ceph-authtool --print-key /var/lib/ceph/osd/ceph-{osd_name}/keyring --name osd.{osd_name}"
+        ).strip()
+        monA.succeed(
+            f"echo '{{\"cephx_secret\": \"{key}\"}}' | ceph osd new {osd_uuid} -i -"
+        )
+
     # Initialize the OSDs with regular filestore
     monA.succeed(
         "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
diff --git a/nixos/tests/ceph-single-node-deprecated-filestore.nix b/nixos/tests/ceph-single-node-deprecated-filestore.nix
deleted file mode 100644
index c08238559155..000000000000
--- a/nixos/tests/ceph-single-node-deprecated-filestore.nix
+++ /dev/null
@@ -1,288 +0,0 @@
-{ lib, ... }:
-
-let
-  cfg = {
-    clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03";
-    monA = {
-      name = "a";
-      ip = "192.168.1.1";
-    };
-    osd0 = {
-      name = "0";
-      key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
-      uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
-    };
-    osd1 = {
-      name = "1";
-      key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
-      uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
-    };
-    osd2 = {
-      name = "2";
-      key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
-      uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
-    };
-  };
-  generateCephConfig =
-    { daemonConfig }:
-    {
-      enable = true;
-      global = {
-        fsid = cfg.clusterId;
-        monHost = cfg.monA.ip;
-        monInitialMembers = cfg.monA.name;
-      };
-    }
-    // daemonConfig;
-
-  generateHost =
-    {
-      cephConfig,
-      networkConfig,
-    }:
-    { pkgs, ... }:
-    {
-      virtualisation = {
-        memorySize = 2048;
-        emptyDiskImages = [
-          20480
-          20480
-          20480
-        ];
-        vlans = [ 1 ];
-      };
-
-      networking = networkConfig;
-
-      environment.systemPackages = with pkgs; [
-        bash
-        sudo
-        ceph
-        xfsprogs
-      ];
-
-      boot.kernelModules = [ "xfs" ];
-
-      services.ceph = cephConfig;
-    };
-
-  networkMonA = {
-    dhcpcd.enable = false;
-    interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
-      {
-        address = cfg.monA.ip;
-        prefixLength = 24;
-      }
-    ];
-  };
-  cephConfigMonA = generateCephConfig {
-    daemonConfig = {
-      mon = {
-        enable = true;
-        daemons = [ cfg.monA.name ];
-      };
-      mgr = {
-        enable = true;
-        daemons = [ cfg.monA.name ];
-      };
-      osd = {
-        enable = true;
-        daemons = [
-          cfg.osd0.name
-          cfg.osd1.name
-          cfg.osd2.name
-        ];
-      };
-      rgw = {
-        enable = true;
-        daemons = [ cfg.monA.name ];
-      };
-    };
-  };
-
-  # Following deployment is based on the manual deployment described here:
-  # https://docs.ceph.com/docs/master/install/manual-deployment/
-  # For other ways to deploy a ceph cluster, look at the documentation at
-  # https://docs.ceph.com/docs/master/
-  testScript = ''
-    import json
-
-    start_all()
-
-    monA.wait_for_unit("network.target")
-
-    # Bootstrap ceph-mon daemon
-    monA.succeed(
-        "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
-        "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
-        "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-        "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
-        "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
-        "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
-        "systemctl start ceph-mon-${cfg.monA.name}",
-    )
-    monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
-    monA.succeed("ceph mon enable-msgr2")
-    monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
-
-    # Can't check ceph status until a mon is up
-    monA.succeed("ceph -s | grep 'mon: 1 daemons'")
-
-    # Start the ceph-mgr daemon, after copying in the keyring
-    monA.succeed(
-        "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
-        "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
-        "systemctl start ceph-mgr-${cfg.monA.name}",
-    )
-    monA.wait_for_unit("ceph-mgr-a")
-    monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-    monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-
-    # Bootstrap OSDs
-    monA.succeed(
-        "mkfs.xfs /dev/vdb",
-        "mkfs.xfs /dev/vdc",
-        "mkfs.xfs /dev/vdd",
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-        "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-        "mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-        "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
-        "mount /dev/vdd /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
-        "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
-        'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
-        'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
-        'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
-    )
-
-    # Initialize the OSDs with regular filestore
-    monA.succeed(
-        "ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
-        "ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
-        "ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
-        "chown -R ceph:ceph /var/lib/ceph/osd",
-        "systemctl start ceph-osd-${cfg.osd0.name}",
-        "systemctl start ceph-osd-${cfg.osd1.name}",
-        "systemctl start ceph-osd-${cfg.osd2.name}",
-    )
-    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-    monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-
-    monA.succeed(
-        "ceph osd pool create single-node-test 32 32",
-        "ceph osd pool ls | grep 'single-node-test'",
-
-        # We need to enable an application on the pool, otherwise it will
-        # stay unhealthy in state POOL_APP_NOT_ENABLED.
-        # Creating a CephFS would do this automatically, but we haven't done that here.
-        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
-        # We use the custom application name "nixos-test" for this.
-        "ceph osd pool application enable single-node-test nixos-test",
-
-        "ceph osd pool rename single-node-test single-node-other-test",
-        "ceph osd pool ls | grep 'single-node-other-test'",
-    )
-    monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
-    monA.succeed(
-        "ceph osd getcrushmap -o crush",
-        "crushtool -d crush -o decrushed",
-        "sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush",
-        "crushtool -c modcrush -o recrushed",
-        "ceph osd setcrushmap -i recrushed",
-        "ceph osd pool set single-node-other-test size 2",
-    )
-    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-    monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
-    monA.fail(
-        "ceph osd pool ls | grep 'multi-node-test'",
-        "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
-    )
-
-    # Bootstrap RGW
-    monA.succeed(
-        "sudo -u ceph mkdir -p /var/lib/ceph/radosgw/ceph-${cfg.monA.name}",
-        "ceph auth get-or-create client.${cfg.monA.name} osd 'allow rwx' mon 'allow rw' > /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
-        "chown ceph:ceph /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
-        "systemctl start ceph-rgw-${cfg.monA.name}",
-    )
-    monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
-    monA.wait_for_open_port(7480)
-
-    # Shut down ceph by stopping ceph.target.
-    monA.succeed("systemctl stop ceph.target")
-
-    # Start it up
-    monA.succeed("systemctl start ceph.target")
-    monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
-    monA.wait_for_unit("ceph-mgr-${cfg.monA.name}")
-    monA.wait_for_unit("ceph-osd-${cfg.osd0.name}")
-    monA.wait_for_unit("ceph-osd-${cfg.osd1.name}")
-    monA.wait_for_unit("ceph-osd-${cfg.osd2.name}")
-    monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
-
-    # Ensure the cluster comes back up again
-    monA.succeed("ceph -s | grep 'mon: 1 daemons'")
-    monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
-    monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
-    monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
-    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-
-    # Enable the dashboard and recheck health
-    monA.succeed(
-        "ceph mgr module enable dashboard",
-        "ceph config set mgr mgr/dashboard/ssl false",
-        # default is 8080 but it's better to be explicit
-        "ceph config set mgr mgr/dashboard/server_port 8080",
-    )
-    monA.wait_for_open_port(8080)
-    monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
-    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-
-    # Initialize dashboard creds
-    monA.succeed(
-        "echo 'foo bar baz qux' > /tmp/dashboard_pw",
-        "ceph dashboard ac-user-create admin -i /tmp/dashboard_pw administrator",
-        "ceph dashboard set-rgw-credentials",
-    )
-
-    # Get dashboard auth token
-    auth_payload = json.dumps({"username": "admin", "password": "foo bar baz qux"})
-    auth_response = json.loads(monA.succeed(
-        f"curl --fail -s -X POST -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -d '{auth_payload}' http://localhost:8080/api/auth",
-    ))
-    token = auth_response["token"]
-
-    # Check cluster health via dashboard API
-    health = json.loads(monA.succeed(
-        f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/health/minimal",
-    ))
-    assert health["health"]["status"] == "HEALTH_OK"
-
-    # List daemons via REST API
-    rgw_daemons = json.loads(monA.succeed(
-        f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/rgw/daemon",
-    ))
-    assert rgw_daemons[0]["id"] == "a"
-  '';
-in
-{
-  name = "basic-single-node-ceph-cluster-deprecated-filestore";
-  meta = with lib.maintainers; {
-    maintainers = [
-      lejonet
-      johanot
-    ];
-  };
-
-  nodes = {
-    monA = generateHost {
-      cephConfig = cephConfigMonA;
-      networkConfig = networkMonA;
-    };
-  };
-
-  inherit testScript;
-}
diff --git a/pkgs/by-name/ce/ceph/ceph.nix b/pkgs/by-name/ce/ceph/ceph.nix
index 1b1c208d693d..6c29e9cf0c21 100644
--- a/pkgs/by-name/ce/ceph/ceph.nix
+++ b/pkgs/by-name/ce/ceph/ceph.nix
@@ -395,10 +395,7 @@ stdenv.mkDerivation {
       inherit (nixosTests)
         ceph-multi-node-bluestore
         ceph-multi-node-bluestore-cephfs
-        ceph-multi-node-deprecated-filestore
         ceph-single-node-bluestore
-        ceph-single-node-bluestore-dmcrypt
-        ceph-single-node-deprecated-filestore
         ;
     };
   };
diff --git a/pkgs/by-name/ce/ceph/src.nix b/pkgs/by-name/ce/ceph/src.nix
index f4605b7f74a7..3844c6284099 100644
--- a/pkgs/by-name/ce/ceph/src.nix
+++ b/pkgs/by-name/ce/ceph/src.nix
@@ -6,11 +6,11 @@
 
 applyPatches (final: {
   pname = "ceph-src";
-  version = "20.2.3";
+  version = "20.2.4";
 
   src = fetchurl {
     url = "https://download.ceph.com/tarballs/ceph-${final.version}.tar.gz";
-    hash = "sha256-y3bZm2lkHiebXYNbZA7jN4VXCLaDEElYvpyuglLISi0=";
+    hash = "sha256-XzRWkkGiiQRGuTHwbNhE+TvKZl90CiBjFCnS1Vsemzc=";
   };
 
   patches = [