nathan/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2026-10-01 20:45:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Backport release-26.05] ceph: 20.2.3 -> 20.2.4 - #560576 (#567843)

Browse Source
This commit is contained in:
Niklas Hambüchen
2026-10-01 12:29:01 +00:00
committed by GitHub
parent 6a5fdf8669 5115a6def0
commit 6aa11f150e
8 changed files with 213 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
nixos/tests/all-tests.nix
View File

@@ -340,22 +340,10 @@ in
ceph-multi-node-bluestore-cephfs = runTestOn [ "aarch64-linux" "x86_64-linux" ] (
import ./ceph-multi-node-bluestore.nix { withCephfs = true; }
);
ceph-multi-node-deprecated-filestore = runTestOn [
"aarch64-linux"
"x86_64-linux"
] ./ceph-multi-node-deprecated-filestore.nix;
ceph-single-node-bluestore = runTestOn [
"aarch64-linux"
"x86_64-linux"
] ./ceph-single-node-bluestore.nix;
ceph-single-node-bluestore-dmcrypt = runTestOn [
"aarch64-linux"
"x86_64-linux"
] ./ceph-single-node-bluestore-dmcrypt.nix;
ceph-single-node-deprecated-filestore = runTestOn [
"aarch64-linux"
"x86_64-linux"
] ./ceph-single-node-deprecated-filestore.nix;
certmgr = import ./certmgr.nix { inherit pkgs runTest; };
cfssl = runTestOn [ "aarch64-linux" "x86_64-linux" ] ./cfssl.nix;
cgit = runTest ./cgit.nix;

244
nixos/tests/ceph-multi-node-bluestore.nix
View File

@@ -25,19 +25,16 @@ let
osd0 = {
name = "0";
ip = "192.168.1.2";
key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
};
osd1 = {
name = "1";
ip = "192.168.1.3";
key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
};
osd2 = {
name = "2";
ip = "192.168.1.4";
key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
};
# Client that mounts CephFS using the in-kernel client.
@@ -58,6 +55,14 @@ let
monHost = cfg.monA.ip;
monInitialMembers = cfg.monA.name;
};
extraConfig = {
log_to_syslog = "false";
log_to_file = "false";
log_to_stderr = "true";
debug_rocksdb = "1/5";
debug_mgr = "1/5";
mon_host = "v2:${cfg.monA.ip}:3300 v1:${cfg.monA.ip}:6789";
};
}
// daemonConfig;
@@ -81,6 +86,7 @@ let
bash
sudo
ceph
cryptsetup
netcat
];
@@ -145,6 +151,11 @@ let
enable = true;
daemons = [ cfg.monA.name ];
};
# TODO: move this to a separate machine
rgw = {
enable = true;
daemons = [ cfg.monA.name ];
};
}
# The MDS daemon (which provides CephFS) is only configured in the CephFS
# variant of this test.
@@ -209,6 +220,11 @@ let
vlans = [ 1 ];
};
# Ceph 20.2.4 introduced the aes256k cipher for authentication.
# Linux started supporting these in kernel version 7.0.
# Remove this line at the earliest convenience (i.e. when tests are run by 7.0 or higher by default).
boot.kernelPackages = pkgs.linuxPackages_latest;
networking = networkConfig;
environment.systemPackages = with pkgs; [
@@ -285,6 +301,8 @@ let
# Based on the "manual deployment" approach from:
# https://docs.ceph.com/en/tentacle/install/manual-deployment/
baseScript = ''
import json
start_all()
monA.wait_for_unit("network.target")
@@ -297,14 +315,15 @@ let
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
"sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
# Creating the mon with v2 (and a legacy v1) address right away removes the need for running `enable-msgr2` later on.
# It is also makes the test more consistent by fixing the address to a known value instead of letting it derive the address.
"monmaptool --create --addv ${cfg.monA.name} '[v2:${cfg.monA.ip}:3300,v1:${cfg.monA.ip}:6789]' --auth-allowed-ciphers aes256k --auth-preferred-cipher aes256k --auth-service-cipher aes256k --fsid ${cfg.clusterId} /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
"systemctl start ceph-mon-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.succeed("ceph mon enable-msgr2")
monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
# Can't check ceph status until a mon is up
@@ -320,59 +339,63 @@ let
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
# Send the admin keyring to the OSD machines.
monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared")
osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
# Send the bootstrap-osd keyring to the OSD machines.
monA.succeed("ceph auth get client.bootstrap-osd -o /etc/ceph/ceph.client.bootstrap-osd.keyring")
monA.succeed("cp /etc/ceph/ceph.client.bootstrap-osd.keyring /tmp/shared")
# Bootstrap the BlueStore OSDs.
osd0.succeed(
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd0.name}/type",
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}/block",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
)
osd1.succeed(
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd1.name}/type",
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}/block",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
)
osd2.succeed(
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type",
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
)
#
# The steps for this are roughly the same for all OSDs:
# 1. get the bootstrap-osd keyring
# 2. prepare the osd via ceph-volume lvm, the second line contains the OSD specific configuration
# 3. deactivate it to unmount the tmpfs
# 4. activate it without a tmpfs for persistent data
# 5. sync, so the osd has at least one consistent state saved
# 6. start it
# We `sync` so that the config survives the forced crashes below.
# osd.0: plain
osd0.succeed(
"ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"mkdir -p /var/lib/ceph/bootstrap-osd",
"cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
"ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-id ${cfg.osd0.name} --osd-fsid ${cfg.osd0.uuid} "
"--data /dev/vdb",
"ceph-volume lvm deactivate ${cfg.osd0.name} ${cfg.osd0.uuid}",
"ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd0.name} ${cfg.osd0.uuid}",
"sync",
"systemctl start ceph-osd-${cfg.osd0.name}",
)
# osd.1: plain
osd1.succeed(
"ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"mkdir -p /var/lib/ceph/bootstrap-osd",
"cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
"ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-id ${cfg.osd1.name} --osd-fsid ${cfg.osd1.uuid} "
"--data /dev/vdb --dmcrypt",
"ceph-volume lvm deactivate ${cfg.osd1.name} ${cfg.osd1.uuid}",
"ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
"sync",
"systemctl start ceph-osd-${cfg.osd1.name}",
)
# osd.2: plain
osd2.succeed(
"ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"mkdir -p /var/lib/ceph/bootstrap-osd",
"cp /tmp/shared/ceph.client.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
"ceph-volume lvm prepare --objectstore bluestore --no-systemd --osd-fsid ${cfg.osd2.uuid} --osd-id ${cfg.osd2.name} "
"--data /dev/vdb",
"ceph-volume lvm deactivate ${cfg.osd2.name} ${cfg.osd2.uuid}",
"ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd2.name} ${cfg.osd2.uuid}",
"sync",
"systemctl start ceph-osd-${cfg.osd2.name}",
)
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.succeed(
# Autoscaling will cause PGs to be peering, causing the tests to become flakey.
"ceph osd pool set noautoscale",
"ceph osd pool create multi-node-test 32 32",
"ceph osd pool ls | grep 'multi-node-test'",
@@ -389,6 +412,7 @@ let
"ceph osd pool ls | grep 'multi-node-other-test'",
)
monA.succeed("ceph osd pool set multi-node-other-test size 2")
# TODO: actually write to the pool using rados directly
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'")
monA.fail(
@@ -396,23 +420,100 @@ let
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",
)
# Bootstrap RGW
monA.succeed(
"sudo -u ceph mkdir -p /var/lib/ceph/radosgw/ceph-${cfg.monA.name}",
"ceph auth get-or-create client.${cfg.monA.name} osd 'allow rwx' mon 'allow rw' > /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
"chown ceph:ceph /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
"systemctl start ceph-rgw-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
monA.wait_for_open_port(7480)
# Enable the dashboard and recheck health
monA.succeed(
"ceph mgr module enable dashboard",
"ceph config set mgr mgr/dashboard/ssl false",
# default is 8080 but it's better to be explicit
"ceph config set mgr mgr/dashboard/server_port 8080",
)
# The dashboard does not listen on localhost:
# `server_addr` defaults to the wildcard address, but the dashboard module
# resolves that to the active mgr's own IP and binds only to it,
# so loopback is never bound.
# See https://github.com/ceph/ceph/blob/v20.2.2/src/pybind/mgr/dashboard/module.py#L213-L214
# Therefore address the dashboard via the mgr's IP instead of localhost.
dashboard = "http://${cfg.monA.ip}:8080"
monA.wait_for_open_port(8080, addr="${cfg.monA.ip}")
monA.wait_until_succeeds(f"curl -s --fail {dashboard}")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# Initialize dashboard creds.
# In a the query below, we test the Dashboard's `/api/rgw/daemon`,
# which needs that the dashboard can talk to RGW.
# `set-rgw-credentials` needs a running RGW daemon.
monA.succeed(
"echo 'foo bar baz qux' > /tmp/dashboard_pw",
"ceph dashboard ac-user-create admin -i /tmp/dashboard_pw administrator",
"ceph dashboard set-rgw-credentials",
"sync",
)
# Get dashboard auth token
auth_payload = json.dumps({"username": "admin", "password": "foo bar baz qux"})
auth_response = json.loads(monA.succeed(
f"curl --fail -s -X POST -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -d '{auth_payload}' {dashboard}/api/auth",
))
token = auth_response["token"]
# Check cluster health via dashboard API
health = json.loads(monA.succeed(
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' {dashboard}/api/health/minimal",
))
assert health["health"]["status"] == "HEALTH_OK"
# List daemons via REST API.
# This also requires a running RGW daemon, as it asserts on the first one.
rgw_daemons = json.loads(monA.succeed(
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' {dashboard}/api/rgw/daemon",
))
assert rgw_daemons[0]["id"] == "${cfg.monA.name}"
# Shut down ceph on all machines in a very unpolite way
monA.crash()
osd0.crash()
osd1.crash()
osd2.crash()
# Start it up
# Start the mon first and mark the OSDs as down.
# Since the heartbeats are pretty high by default, the OSDs would otherwise be marked as up still.
# However we do not want to lower the heartbeats since this might cause flakey tests.
monA.start()
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.wait_until_succeeds("ceph osd down all")
# Then start the OSDs as normal.
osd0.start()
osd1.start()
osd2.start()
monA.start()
# Ensure they are all up.
osd0.wait_for_unit("network.target")
osd1.wait_for_unit("network.target")
osd2.wait_for_unit("network.target")
# Ensure the cluster comes back up again.
# FIXME: dmcrypt OSDs currently do not work out of the box.
# For a potential long-term fix see: https://github.com/NixOS/nixpkgs/pull/512912#discussion_r3140295546
osd1.succeed(
"ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
"systemctl start ceph-osd-${cfg.osd1.name}",
)
# Test the cluster state thoroughly.
monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# Verify the recovery.
@@ -444,45 +545,50 @@ let
# Create a CephFS.
monA.succeed(
"ceph osd pool create cephfs-data 32 32",
"ceph osd pool create cephfs-metadata 32 32",
"ceph fs new cephfs cephfs-metadata cephfs-data",
"ceph fs volume create testing",
"ceph osd pool set cephfs.testing.data pg_num 32",
"ceph osd pool set cephfs.testing.meta pg_num 32",
)
# Wait for the MDS to claim the filesystem and become active.
monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
monA.wait_until_succeeds("ceph fs status testing | grep -e 'active'", timeout=60)
# Distribute the admin keyring (and a plain secret file for the kernel
# client) to both client machines, so that they can authenticate.
# Create a subvolume, issue credentials, then distribute those credentials.
monA.succeed(
"cp /etc/ceph/ceph.client.admin.keyring /tmp/shared",
"ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > /tmp/shared/admin.secret",
"ceph fs subvolumegroup create testing group",
"ceph fs subvolume create testing subvolume --group_name group",
"ceph fs subvolume authorize testing subvolume kclient group",
"ceph fs subvolume authorize testing subvolume fuseclient group",
"ceph auth get client.kclient -o /tmp/shared/ceph.client.kclient.keyring",
"ceph auth get client.fuseclient -o /tmp/shared/ceph.client.fuseclient.keyring",
)
kclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
fuseclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
kclient.succeed("cp /tmp/shared/admin.secret /etc/ceph/admin.secret")
kclient.succeed("cp /tmp/shared/ceph.client.kclient.keyring /etc/ceph")
fuseclient.succeed("cp /tmp/shared/ceph.client.fuseclient.keyring /etc/ceph")
# Get the volume path generated by Ceph.
volume_path = monA.succeed("ceph fs subvolume getpath testing subvolume group | tee /dev/stderr").strip()
# Mount CephFS on the kernel client.
# We force the messenger v2 protocol via "ms_mode=secure"; the cluster
# has msgr2 enabled (see "ceph mon enable-msgr2" above) and the legacy v1
# has msgr2 enabled (the monmap is created with a v2 address above) and the legacy v1
# protocol apparently does not reconnect reliably after the servers are restarted.
# The msgr2 monitor listens on port 3300 (instead of legacy v1 port 6789),
# so we have to point the device string at that port explicitly.
# `recover_session=clean` makes the kernel client automatically reconnect
# (discarding its stale session) after the whole cluster has been down,
# which would otherwise leave the mount blocklisted and hanging forever.
# which would otherwise leave the mount blocklisted and hanging.
# Real CephFS use may not prefer hanging `recover_session=clean`, and
# prefer manual de-blocklisting to avoid any failed OS syscalls,
# but for this test, discarding stale sessions is good enough.
kclient.succeed("mkdir -p /mnt/cephfs")
kclient.wait_until_succeeds(
"mount -t ceph ${cfg.monA.ip}:3300:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret,ms_mode=secure,recover_session=clean"
f"mount -t ceph kclient@.testing={volume_path} /mnt/cephfs -o ms_mode=secure,recover_session=clean"
)
kclient.succeed("mountpoint /mnt/cephfs")
# Mount CephFS on the FUSE client using ceph-fuse.
fuseclient.succeed("mkdir -p /mnt/cephfs")
fuseclient.wait_until_succeeds(
"ceph-fuse --id admin -m ${cfg.monA.ip}:6789 /mnt/cephfs"
f"ceph-fuse --id fuseclient -m ${cfg.monA.ip}:3300 -r {volume_path} /mnt/cephfs"
)
fuseclient.succeed("mountpoint /mnt/cephfs")
@@ -510,24 +616,40 @@ let
osd1.crash()
osd2.crash()
# Start it up
# Start the mon first and mark the OSDs as down.
# Since the heartbeats are pretty high by default, the OSDs would otherwise be marked as up still.
# However we do not want to lower the heartbeats since this might cause flakey tests.
monA.start()
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.wait_until_succeeds("ceph osd down all")
# Then start the OSDs as normal.
osd0.start()
osd1.start()
osd2.start()
monA.start()
# Ensure they are all up.
osd0.wait_for_unit("network.target")
osd1.wait_for_unit("network.target")
osd2.wait_for_unit("network.target")
# FIXME: dmcrypt OSDs currently do not work out of the box.
# For a potential long-term fix see: https://github.com/NixOS/nixpkgs/pull/512912#discussion_r3140295546
osd1.succeed(
"ceph-volume lvm activate --no-tmpfs --no-systemd ${cfg.osd1.name} ${cfg.osd1.uuid}",
"systemctl start ceph-osd-${cfg.osd1.name}",
)
# Ensure the cluster comes back up again.
# See the note above on why this uses `wait_until_succeeds`.
monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'", timeout=60)
# Ensure the MDS/CephFS comes back up again, too.
monA.wait_for_unit("ceph-mds-${cfg.monA.name}")
monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
monA.wait_until_succeeds("ceph fs status testing | grep -e 'active'", timeout=60)
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# The clients kept running across the outage, so their CephFS mounts

291
nixos/tests/ceph-multi-node-deprecated-filestore.nix
View File

@@ -1,291 +0,0 @@
# Tests the legacy FileStore OSD backend.
{ lib, ... }:
let
cfg = {
clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03";
monA = {
name = "a";
ip = "192.168.1.1";
};
osd0 = {
name = "0";
ip = "192.168.1.2";
key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
};
osd1 = {
name = "1";
ip = "192.168.1.3";
key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
};
osd2 = {
name = "2";
ip = "192.168.1.4";
key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
};
};
generateCephConfig =
{ daemonConfig }:
{
enable = true;
global = {
fsid = cfg.clusterId;
monHost = cfg.monA.ip;
monInitialMembers = cfg.monA.name;
};
}
// daemonConfig;
generateHost =
{ cephConfig, networkConfig }:
{ pkgs, ... }:
{
virtualisation = {
emptyDiskImages = [ 20480 ];
vlans = [ 1 ];
};
networking = networkConfig;
environment.systemPackages = with pkgs; [
bash
sudo
ceph
xfsprogs
netcat
];
boot.kernelModules = [ "xfs" ];
services.ceph = cephConfig;
};
networkMonA = {
dhcpcd.enable = false;
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
{
address = cfg.monA.ip;
prefixLength = 24;
}
];
firewall = {
allowedTCPPorts = [
6789
3300
];
allowedTCPPortRanges = [
{
from = 6800;
to = 7300;
}
];
};
};
cephConfigMonA = generateCephConfig {
daemonConfig = {
mon = {
enable = true;
daemons = [ cfg.monA.name ];
};
mgr = {
enable = true;
daemons = [ cfg.monA.name ];
};
};
};
networkOsd = osd: {
dhcpcd.enable = false;
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
{
address = osd.ip;
prefixLength = 24;
}
];
firewall = {
allowedTCPPortRanges = [
{
from = 6800;
to = 7300;
}
];
};
};
cephConfigOsd =
osd:
generateCephConfig {
daemonConfig = {
osd = {
enable = true;
daemons = [ osd.name ];
};
};
};
# Following deployment is based on the manual deployment described here:
# https://docs.ceph.com/docs/master/install/manual-deployment/
# For other ways to deploy a ceph cluster, look at the documentation at
# https://docs.ceph.com/docs/master/
testscript =
{ ... }:
''
start_all()
monA.wait_for_unit("network.target")
osd0.wait_for_unit("network.target")
osd1.wait_for_unit("network.target")
osd2.wait_for_unit("network.target")
# Bootstrap ceph-mon daemon
monA.succeed(
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
"sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
"systemctl start ceph-mon-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.succeed("ceph mon enable-msgr2")
monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
# Can't check ceph status until a mon is up
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
# Start the ceph-mgr daemon, it has no deps and hardly any setup
monA.succeed(
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
"sync", # to ensure shell redirection above is durable
"systemctl start ceph-mgr-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mgr-a")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
# Send the admin keyring to the OSD machines
monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared")
osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
# Bootstrap OSDs
osd0.succeed(
"mkfs.xfs /dev/vdb",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
)
osd1.succeed(
"mkfs.xfs /dev/vdb",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
)
osd2.succeed(
"mkfs.xfs /dev/vdb",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
)
# We `sync` so that the config survives the forced crashes below.
osd0.succeed(
"ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"sync",
"systemctl start ceph-osd-${cfg.osd0.name}",
)
osd1.succeed(
"ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"sync",
"systemctl start ceph-osd-${cfg.osd1.name}",
)
osd2.succeed(
"ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"sync",
"systemctl start ceph-osd-${cfg.osd2.name}",
)
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.succeed(
"ceph osd pool create multi-node-test 32 32",
"ceph osd pool ls | grep 'multi-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable multi-node-test nixos-test",
"ceph osd pool rename multi-node-test multi-node-other-test",
"ceph osd pool ls | grep 'multi-node-other-test'",
)
monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
monA.succeed("ceph osd pool set multi-node-other-test size 2")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
monA.fail(
"ceph osd pool ls | grep 'multi-node-test'",
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",
)
# Shut down ceph on all machines in a very unpolite way
monA.crash()
osd0.crash()
osd1.crash()
osd2.crash()
# Start it up
osd0.start()
osd1.start()
osd2.start()
monA.start()
# Ensure the cluster comes back up again
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
'';
in
{
name = "basic-multi-node-ceph-cluster-deprecated-filestore";
meta = with lib.maintainers; {
maintainers = [ lejonet ];
};
nodes = {
monA = generateHost {
cephConfig = cephConfigMonA;
networkConfig = networkMonA;
};
osd0 = generateHost {
cephConfig = cephConfigOsd cfg.osd0;
networkConfig = networkOsd cfg.osd0;
};
osd1 = generateHost {
cephConfig = cephConfigOsd cfg.osd1;
networkConfig = networkOsd cfg.osd1;
};
osd2 = generateHost {
cephConfig = cephConfigOsd cfg.osd2;
networkConfig = networkOsd cfg.osd2;
};
};
testScript = testscript;
}

269
nixos/tests/ceph-single-node-bluestore-dmcrypt.nix
View File

@@ -1,269 +0,0 @@
{ lib, ... }:
let
# the single node ipv6 address
ip = "2001:db8:ffff::";
# the global ceph cluster id
cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";
# the fsids of OSDs
osd-fsid-map = {
"0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";
"1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";
"2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";
};
in
{
name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";
meta.maintainers = with lib.maintainers; [
benaryorg
nh2
];
nodes.ceph =
{
lib,
pkgs,
config,
...
}:
{
# disks for bluestore
virtualisation.emptyDiskImages = [
20480
20480
20480
];
# networking setup (no external connectivity required, only local IPv6)
networking.useDHCP = false;
systemd.network = {
enable = true;
wait-online.extraArgs = [
"-i"
"lo"
];
networks = {
"40-loopback" = {
enable = true;
name = "lo";
DHCP = "no";
addresses = [ { Address = "${ip}/128"; } ];
};
};
};
# do not start the ceph target by default so we can format the disks first
systemd.targets.ceph.wantedBy = lib.mkForce [ ];
# add the packages to systemPackages so the testscript doesn't run into any unexpected issues
# this shouldn't be required on production systems which have their required packages in the unit paths only
# but it helps in case one needs to actually run the tooling anyway
environment.systemPackages = with pkgs; [
ceph
cryptsetup
lvm2
];
services.ceph = {
enable = true;
client.enable = true;
extraConfig = {
public_addr = ip;
cluster_addr = ip;
# ipv6
ms_bind_ipv4 = "false";
ms_bind_ipv6 = "true";
# msgr2 settings
ms_cluster_mode = "secure";
ms_service_mode = "secure";
ms_client_mode = "secure";
ms_mon_cluster_mode = "secure";
ms_mon_service_mode = "secure";
ms_mon_client_mode = "secure";
# less default modules, cuts down on memory and startup time in the tests
mgr_initial_modules = "";
# distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host
osd_crush_chooseleaf_type = "0";
};
client.extraConfig."mon.0" = {
host = "ceph";
mon_addr = "v2:[${ip}]:3300";
public_addr = "v2:[${ip}]:3300";
};
global = {
fsid = cluster;
clusterNetwork = "${ip}/64";
publicNetwork = "${ip}/64";
monInitialMembers = "0";
};
mon = {
enable = true;
daemons = [ "0" ];
};
osd = {
enable = true;
daemons = builtins.attrNames osd-fsid-map;
};
mgr = {
enable = true;
daemons = [ "ceph" ];
};
};
systemd.services =
let
osd-name = id: "ceph-osd-${id}";
osd-pre-start = id: [
"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"
"${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"
];
osd-post-stop = id: [
"!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"
];
map-osd = id: {
name = osd-name id;
value = {
serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);
serviceConfig.ExecStopPost = osd-post-stop id;
unitConfig.ConditionPathExists = lib.mkForce [ ];
unitConfig.StartLimitBurst = lib.mkForce 4;
path = with pkgs; [
util-linux
lvm2
cryptsetup
];
};
};
in
lib.pipe config.services.ceph.osd.daemons [
(map map-osd)
builtins.listToAttrs
];
};
testScript = ''
start_all()
ceph.wait_for_unit("default.target")
# Bootstrap ceph-mon daemon
ceph.succeed(
"mkdir -p /var/lib/ceph/bootstrap-osd",
"ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
"ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
"ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",
"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
"monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",
"mkdir -p /var/lib/ceph/mon/ceph-0",
"ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",
"chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",
"systemctl start ceph-mon-0.service",
)
ceph.wait_for_unit("ceph-mon-0.service")
# should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout
ceph.wait_for_open_port(3300, "${ip}")
ceph.succeed(
# required for HEALTH_OK
"ceph config set mon auth_allow_insecure_global_id_reclaim false",
# IPv6
"ceph config set global ms_bind_ipv4 false",
"ceph config set global ms_bind_ipv6 true",
# the new (secure) protocol
"ceph config set global ms_bind_msgr1 false",
"ceph config set global ms_bind_msgr2 true",
# just a small little thing
"ceph config set mon mon_compact_on_start true",
)
# Can't check ceph status until a mon is up
ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
# Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)
ceph.succeed(
# this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases
"ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",
"ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",
"ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",
"sudo ceph-volume lvm deactivate 0",
"sudo ceph-volume lvm deactivate 1",
"sudo ceph-volume lvm deactivate 2",
"chown -R ceph:ceph /var/lib/ceph",
)
# Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)
ceph.succeed(
"systemctl start ceph-osd-0.service",
"systemctl start ceph-osd-1.service",
"systemctl start ceph-osd-2.service",
)
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
# Start the ceph-mgr daemon, after copying in the keyring
ceph.succeed(
"mkdir -p /var/lib/ceph/mgr/ceph-ceph/",
"ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",
"chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",
"systemctl start ceph-mgr-ceph.service",
)
ceph.wait_for_unit("ceph-mgr-ceph")
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# test the actual storage
ceph.succeed(
"ceph osd pool create single-node-test 32 32",
"ceph osd pool ls | grep 'single-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable single-node-test nixos-test",
"ceph osd pool rename single-node-test single-node-other-test",
"ceph osd pool ls | grep 'single-node-other-test'",
)
ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")
ceph.fail(
# the old pool should be gone
"ceph osd pool ls | grep 'multi-node-test'",
# deleting the pool should fail without setting mon_allow_pool_delete
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
)
# rebooting gets rid of any potential tmpfs mounts or device-mapper devices
ceph.shutdown()
ceph.start()
ceph.wait_for_unit("default.target")
# Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)
ceph.systemctl("start ceph-mon-0.service")
ceph.wait_for_unit("ceph-mon-0")
ceph.systemctl("start ceph-osd-0.service")
ceph.wait_for_unit("ceph-osd-0")
ceph.systemctl("start ceph-osd-1.service")
ceph.wait_for_unit("ceph-osd-1")
ceph.systemctl("start ceph-osd-2.service")
ceph.wait_for_unit("ceph-osd-2")
ceph.systemctl("start ceph-mgr-ceph.service")
ceph.wait_for_unit("ceph-mgr-ceph")
# Ensure the cluster comes back up again
ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")
ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
'';
}

39
nixos/tests/ceph-single-node-bluestore.nix
View File

@@ -9,17 +9,14 @@ let
};
osd0 = {
name = "0";
key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
};
osd1 = {
name = "1";
key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
};
osd2 = {
name = "2";
key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
};
};
@@ -51,6 +48,11 @@ let
vlans = [ 1 ];
};
# Ceph 20.2.4 introduced the aes256k cipher for authentication.
# Linux started supporting these in kernel version 7.0.
# Remove this line at the earliest convenience (i.e. when tests are run by 7.0 or higher by default).
boot.kernelPackages = pkgs.linuxPackages_latest;
networking = networkConfig;
environment.systemPackages = with pkgs; [
@@ -109,13 +111,18 @@ let
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
"sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
# Create the monmap with both a msgr2 (v2) and a legacy (v1) address.
# Using plain `--add` yields a v1-only monmap, which leaves the cluster
# in HEALTH_WARN with MON_MSGR2_NOT_ENABLED. Running `ceph mon
# enable-msgr2` afterwards is not enough: it rewrites the monmap (a
# subsequent `ceph mon dump` does show the v2 address), but the health
# check keeps reporting the mon as v1-only indefinitely.
"monmaptool --create --addv ${cfg.monA.name} '[v2:${cfg.monA.ip}:3300,v1:${cfg.monA.ip}:6789]' --fsid ${cfg.clusterId} /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
"systemctl start ceph-mon-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.succeed("ceph mon enable-msgr2")
monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
# Can't check ceph status until a mon is up
@@ -142,14 +149,24 @@ let
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type",
"ln -sf /dev/vdd /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --gen-key",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --gen-key",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --gen-key",
)
# Register the OSDs with the generated keys read back from their keyrings.
for osd_name, osd_uuid in [
("${cfg.osd0.name}", "${cfg.osd0.uuid}"),
("${cfg.osd1.name}", "${cfg.osd1.uuid}"),
("${cfg.osd2.name}", "${cfg.osd2.uuid}"),
]:
key = monA.succeed(
f"ceph-authtool --print-key /var/lib/ceph/osd/ceph-{osd_name}/keyring --name osd.{osd_name}"
).strip()
monA.succeed(
f"echo '{{\"cephx_secret\": \"{key}\"}}' | ceph osd new {osd_uuid} -i -"
)
# Initialize the OSDs with regular filestore
monA.succeed(
"ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",

288
nixos/tests/ceph-single-node-deprecated-filestore.nix
View File

@@ -1,288 +0,0 @@
{ lib, ... }:
let
cfg = {
clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03";
monA = {
name = "a";
ip = "192.168.1.1";
};
osd0 = {
name = "0";
key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
};
osd1 = {
name = "1";
key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
};
osd2 = {
name = "2";
key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
};
};
generateCephConfig =
{ daemonConfig }:
{
enable = true;
global = {
fsid = cfg.clusterId;
monHost = cfg.monA.ip;
monInitialMembers = cfg.monA.name;
};
}
// daemonConfig;
generateHost =
{
cephConfig,
networkConfig,
}:
{ pkgs, ... }:
{
virtualisation = {
memorySize = 2048;
emptyDiskImages = [
20480
20480
20480
];
vlans = [ 1 ];
};
networking = networkConfig;
environment.systemPackages = with pkgs; [
bash
sudo
ceph
xfsprogs
];
boot.kernelModules = [ "xfs" ];
services.ceph = cephConfig;
};
networkMonA = {
dhcpcd.enable = false;
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
{
address = cfg.monA.ip;
prefixLength = 24;
}
];
};
cephConfigMonA = generateCephConfig {
daemonConfig = {
mon = {
enable = true;
daemons = [ cfg.monA.name ];
};
mgr = {
enable = true;
daemons = [ cfg.monA.name ];
};
osd = {
enable = true;
daemons = [
cfg.osd0.name
cfg.osd1.name
cfg.osd2.name
];
};
rgw = {
enable = true;
daemons = [ cfg.monA.name ];
};
};
};
# Following deployment is based on the manual deployment described here:
# https://docs.ceph.com/docs/master/install/manual-deployment/
# For other ways to deploy a ceph cluster, look at the documentation at
# https://docs.ceph.com/docs/master/
testScript = ''
import json
start_all()
monA.wait_for_unit("network.target")
# Bootstrap ceph-mon daemon
monA.succeed(
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
"sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
"systemctl start ceph-mon-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.succeed("ceph mon enable-msgr2")
monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
# Can't check ceph status until a mon is up
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
# Start the ceph-mgr daemon, after copying in the keyring
monA.succeed(
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
"systemctl start ceph-mgr-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-mgr-a")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
# Bootstrap OSDs
monA.succeed(
"mkfs.xfs /dev/vdb",
"mkfs.xfs /dev/vdc",
"mkfs.xfs /dev/vdd",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"mount /dev/vdd /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
)
# Initialize the OSDs with regular filestore
monA.succeed(
"ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
"ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
"ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
"chown -R ceph:ceph /var/lib/ceph/osd",
"systemctl start ceph-osd-${cfg.osd0.name}",
"systemctl start ceph-osd-${cfg.osd1.name}",
"systemctl start ceph-osd-${cfg.osd2.name}",
)
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.succeed(
"ceph osd pool create single-node-test 32 32",
"ceph osd pool ls | grep 'single-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable single-node-test nixos-test",
"ceph osd pool rename single-node-test single-node-other-test",
"ceph osd pool ls | grep 'single-node-other-test'",
)
monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
monA.succeed(
"ceph osd getcrushmap -o crush",
"crushtool -d crush -o decrushed",
"sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush",
"crushtool -c modcrush -o recrushed",
"ceph osd setcrushmap -i recrushed",
"ceph osd pool set single-node-other-test size 2",
)
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
monA.fail(
"ceph osd pool ls | grep 'multi-node-test'",
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
)
# Bootstrap RGW
monA.succeed(
"sudo -u ceph mkdir -p /var/lib/ceph/radosgw/ceph-${cfg.monA.name}",
"ceph auth get-or-create client.${cfg.monA.name} osd 'allow rwx' mon 'allow rw' > /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
"chown ceph:ceph /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
"systemctl start ceph-rgw-${cfg.monA.name}",
)
monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
monA.wait_for_open_port(7480)
# Shut down ceph by stopping ceph.target.
monA.succeed("systemctl stop ceph.target")
# Start it up
monA.succeed("systemctl start ceph.target")
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
monA.wait_for_unit("ceph-mgr-${cfg.monA.name}")
monA.wait_for_unit("ceph-osd-${cfg.osd0.name}")
monA.wait_for_unit("ceph-osd-${cfg.osd1.name}")
monA.wait_for_unit("ceph-osd-${cfg.osd2.name}")
monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
# Ensure the cluster comes back up again
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# Enable the dashboard and recheck health
monA.succeed(
"ceph mgr module enable dashboard",
"ceph config set mgr mgr/dashboard/ssl false",
# default is 8080 but it's better to be explicit
"ceph config set mgr mgr/dashboard/server_port 8080",
)
monA.wait_for_open_port(8080)
monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# Initialize dashboard creds
monA.succeed(
"echo 'foo bar baz qux' > /tmp/dashboard_pw",
"ceph dashboard ac-user-create admin -i /tmp/dashboard_pw administrator",
"ceph dashboard set-rgw-credentials",
)
# Get dashboard auth token
auth_payload = json.dumps({"username": "admin", "password": "foo bar baz qux"})
auth_response = json.loads(monA.succeed(
f"curl --fail -s -X POST -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -d '{auth_payload}' http://localhost:8080/api/auth",
))
token = auth_response["token"]
# Check cluster health via dashboard API
health = json.loads(monA.succeed(
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/health/minimal",
))
assert health["health"]["status"] == "HEALTH_OK"
# List daemons via REST API
rgw_daemons = json.loads(monA.succeed(
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/rgw/daemon",
))
assert rgw_daemons[0]["id"] == "a"
'';
in
{
name = "basic-single-node-ceph-cluster-deprecated-filestore";
meta = with lib.maintainers; {
maintainers = [
lejonet
johanot
];
};
nodes = {
monA = generateHost {
cephConfig = cephConfigMonA;
networkConfig = networkMonA;
};
};
inherit testScript;
}

3
pkgs/by-name/ce/ceph/ceph.nix
View File

@@ -395,10 +395,7 @@ stdenv.mkDerivation {
inherit (nixosTests)
ceph-multi-node-bluestore
ceph-multi-node-bluestore-cephfs
ceph-multi-node-deprecated-filestore
ceph-single-node-bluestore
ceph-single-node-bluestore-dmcrypt
ceph-single-node-deprecated-filestore
;
};
};

4
pkgs/by-name/ce/ceph/src.nix
View File

@@ -6,11 +6,11 @@
applyPatches (final: {
pname = "ceph-src";
version = "20.2.3";
version = "20.2.4";
src = fetchurl {
url = "https://download.ceph.com/tarballs/ceph-${final.version}.tar.gz";
hash = "sha256-y3bZm2lkHiebXYNbZA7jN4VXCLaDEElYvpyuglLISi0=";
hash = "sha256-XzRWkkGiiQRGuTHwbNhE+TvKZl90CiBjFCnS1Vsemzc=";
};
patches = [