mirror of
https://github.com/NixOS/nixpkgs.git
synced 2026-08-31 20:54:56 +00:00
6a69534ea0
This should give us higher confidence that changes don't break real-world clusters. Originally motivated by the fuse2 -> fuse3 change. Assisted-by: Zoo Code with Claude Opus 4.8, human cleanup.
603 lines
22 KiB
Nix
603 lines
22 KiB
Nix
# Multi-node Ceph cluster test using BlueStore OSDs.
|
|
{
|
|
withCephfs ? false,
|
|
}:
|
|
{ lib, ... }:
|
|
let
|
|
# Development knobs:
|
|
# * `defaultTimeout` caps how long every `waitUntilSucceeds` waits before it
|
|
# gives up. Lower it for faster feedback while iterating on the test.
|
|
# * When `investigateFailures` is true, a `waitUntilSucceeds` that times out
|
|
# does NOT fail the test; instead it dumps each machine's full (multi-boot)
|
|
# journal and `/var/log/ceph` into a `test-failure-investigation/` directory
|
|
# inside the test's working/output directory and then lets the test succeed,
|
|
# so the collected logs end up in the build's store path for convenient
|
|
# reading. Keep this `false` for real CI runs.
|
|
defaultTimeout = 60;
|
|
investigateFailures = false;
|
|
|
|
cfg = {
|
|
clusterId = "066ae264-2a5d-4729-8001-6ad265f50b03";
|
|
monA = {
|
|
name = "a";
|
|
ip = "192.168.1.1";
|
|
};
|
|
osd0 = {
|
|
name = "0";
|
|
ip = "192.168.1.2";
|
|
key = "AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==";
|
|
uuid = "55ba2294-3e24-478f-bee0-9dca4c231dd9";
|
|
};
|
|
osd1 = {
|
|
name = "1";
|
|
ip = "192.168.1.3";
|
|
key = "AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==";
|
|
uuid = "5e97a838-85b6-43b0-8950-cb56d554d1e5";
|
|
};
|
|
osd2 = {
|
|
name = "2";
|
|
ip = "192.168.1.4";
|
|
key = "AQAdyhZeIaUlARAAGRoidDAmS6Vkp546UFEf5w==";
|
|
uuid = "ea999274-13d0-4dd5-9af9-ad25a324f72f";
|
|
};
|
|
# Client that mounts CephFS using the in-kernel client.
|
|
kclient = {
|
|
ip = "192.168.1.5";
|
|
};
|
|
# Client that mounts CephFS using the `ceph-fuse` client.
|
|
fuseclient = {
|
|
ip = "192.168.1.6";
|
|
};
|
|
};
|
|
generateCephConfig =
|
|
{ daemonConfig }:
|
|
{
|
|
enable = true;
|
|
global = {
|
|
fsid = cfg.clusterId;
|
|
monHost = cfg.monA.ip;
|
|
monInitialMembers = cfg.monA.name;
|
|
};
|
|
}
|
|
// daemonConfig;
|
|
|
|
generateHost =
|
|
{ cephConfig, networkConfig }:
|
|
{ pkgs, ... }:
|
|
{
|
|
virtualisation = {
|
|
# A single raw block device per machine, consumed directly by BlueStore
|
|
# as `/dev/vdb`. Because BlueStore owns the raw device (there is no
|
|
# filesystem to mount), the OSD's on-disk state survives a hard
|
|
# crash/reboot and the OSD comes back automatically.
|
|
emptyDiskImages = [ 20480 ];
|
|
vlans = [ 1 ];
|
|
};
|
|
|
|
networking = networkConfig;
|
|
|
|
# TODO: Why do we need any of these? Shouldn't Ceph work independent of `systemPackages`? Only `sudo` and `ceph` are used in our own test code.
|
|
environment.systemPackages = with pkgs; [
|
|
bash
|
|
sudo
|
|
ceph
|
|
netcat
|
|
];
|
|
|
|
services.ceph = cephConfig;
|
|
|
|
# Restart limits are unsuitable for daemons that must recover from
|
|
# arbitrary crash/network downtimes.
|
|
# Ensure all daemons have infinite restart limits.
|
|
# Otherwise the tests are flaky based on timing.
|
|
systemd.services =
|
|
let
|
|
daemonUnits =
|
|
lib.concatMap
|
|
(
|
|
daemonType:
|
|
lib.optionals (cephConfig.${daemonType}.enable or false) (
|
|
map (daemon: "ceph-${daemonType}-${daemon}") cephConfig.${daemonType}.daemons
|
|
)
|
|
)
|
|
[
|
|
"mon"
|
|
"mgr"
|
|
"osd"
|
|
"mds"
|
|
];
|
|
in
|
|
lib.genAttrs daemonUnits (_: {
|
|
serviceConfig.Restart = lib.mkForce "always";
|
|
serviceConfig.RestartSec = lib.mkForce "1";
|
|
unitConfig.StartLimitIntervalSec = lib.mkForce 0; # Ensure Restart=always is always honoured (no start limit)
|
|
});
|
|
};
|
|
|
|
networkMonA = {
|
|
dhcpcd.enable = false;
|
|
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
|
|
{
|
|
address = cfg.monA.ip;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
firewall = {
|
|
allowedTCPPorts = [
|
|
6789
|
|
3300
|
|
];
|
|
allowedTCPPortRanges = [
|
|
{
|
|
from = 6800;
|
|
to = 7300;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
cephConfigMonA = generateCephConfig {
|
|
daemonConfig = {
|
|
mon = {
|
|
enable = true;
|
|
daemons = [ cfg.monA.name ];
|
|
};
|
|
mgr = {
|
|
enable = true;
|
|
daemons = [ cfg.monA.name ];
|
|
};
|
|
}
|
|
# The MDS daemon (which provides CephFS) is only configured in the CephFS
|
|
# variant of this test.
|
|
// lib.optionalAttrs withCephfs {
|
|
mds = {
|
|
enable = true;
|
|
daemons = [ cfg.monA.name ];
|
|
};
|
|
};
|
|
};
|
|
|
|
networkOsd = osd: {
|
|
dhcpcd.enable = false;
|
|
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
|
|
{
|
|
address = osd.ip;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
firewall = {
|
|
allowedTCPPortRanges = [
|
|
{
|
|
from = 6800;
|
|
to = 7300;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
cephConfigOsd =
|
|
osd:
|
|
generateCephConfig {
|
|
daemonConfig = {
|
|
osd = {
|
|
enable = true;
|
|
daemons = [ osd.name ];
|
|
};
|
|
};
|
|
};
|
|
|
|
# The CephFS clients only need the ceph client tooling. They do not run any
|
|
# ceph daemon, so they only get a minimal ceph config pointing at the mon.
|
|
# The in-kernel ceph filesystem module is part of the default kernel and is
|
|
# autoloaded by `mount -t ceph`, so no extra modules are needed.
|
|
networkClient = client: {
|
|
dhcpcd.enable = false;
|
|
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [
|
|
{
|
|
address = client.ip;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
};
|
|
|
|
cephConfigClient = generateCephConfig { daemonConfig = { }; };
|
|
|
|
generateClientHost =
|
|
{ networkConfig }:
|
|
{ pkgs, ... }:
|
|
{
|
|
virtualisation = {
|
|
vlans = [ 1 ];
|
|
};
|
|
|
|
networking = networkConfig;
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
ceph
|
|
];
|
|
|
|
services.ceph = cephConfigClient;
|
|
};
|
|
|
|
# Python prelude that must run before any test code.
|
|
# It replaces every machine's `wait_until_succeeds()` with a wrapper that
|
|
# implements the "development knobs" above.
|
|
helperScript = ''
|
|
import os
|
|
import shutil
|
|
|
|
DEFAULT_TIMEOUT = ${toString defaultTimeout}
|
|
INVESTIGATE_FAILURES = ${if investigateFailures then "True" else "False"}
|
|
|
|
def collect_failure_investigation(failed_machine, command):
|
|
out_dir = os.path.join(driver.out_dir, "test-failure-investigation")
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
with open(os.path.join(out_dir, "README.txt"), "w") as f:
|
|
f.write(
|
|
"wait_until_succeeds timed out on machine "
|
|
f"'{failed_machine.name}' running command:\n{command}\n"
|
|
)
|
|
for m in machines:
|
|
try:
|
|
m.execute("journalctl --no-pager --boot=all > /tmp/journal.txt 2>&1; true")
|
|
m.copy_from_machine("/tmp/journal.txt", out_dir)
|
|
shutil.move(
|
|
os.path.join(out_dir, "journal.txt"),
|
|
os.path.join(out_dir, f"{m.name}-journal.txt"),
|
|
)
|
|
except Exception as e:
|
|
m.log(f"could not collect journal: {e}")
|
|
try:
|
|
m.copy_from_machine("/var/log/ceph", out_dir)
|
|
shutil.move(
|
|
os.path.join(out_dir, "ceph"),
|
|
os.path.join(out_dir, f"{m.name}-ceph-logs"),
|
|
)
|
|
except Exception as e:
|
|
m.log(f"could not collect /var/log/ceph: {e}")
|
|
|
|
# Replace the test driver's `wait_until_succeeds` with our wrapper.
|
|
machine_class = machines[0].__class__
|
|
orig_wait_until_succeeds = machine_class.wait_until_succeeds
|
|
|
|
def patched_wait_until_succeeds(self, command: str, timeout: int = DEFAULT_TIMEOUT) -> str:
|
|
try:
|
|
return orig_wait_until_succeeds(self, command, timeout=timeout)
|
|
except Exception:
|
|
if not INVESTIGATE_FAILURES:
|
|
raise
|
|
self.log(
|
|
"wait_until_succeeds timed out; collecting logs into "
|
|
"test-failure-investigation/ and ending the test as 'passed' "
|
|
"(investigateFailures = true)"
|
|
)
|
|
collect_failure_investigation(self, command)
|
|
os._exit(0)
|
|
|
|
# Use `setattr` (rather than a plain attribute assignment) so the type
|
|
# checker does not treat this as implicitly shadowing the driver's
|
|
# `wait_until_succeeds`; the replacement is intentional.
|
|
setattr(machine_class, "wait_until_succeeds", patched_wait_until_succeeds)
|
|
'';
|
|
|
|
# Set up the cluster (mon, mgr, BlueStore OSDs) and perform a
|
|
# hard whole-cluster crash/recovery (failover) test.
|
|
#
|
|
# Based on the "manual deployment" approach from:
|
|
# https://docs.ceph.com/en/tentacle/install/manual-deployment/
|
|
baseScript = ''
|
|
start_all()
|
|
|
|
monA.wait_for_unit("network.target")
|
|
osd0.wait_for_unit("network.target")
|
|
osd1.wait_for_unit("network.target")
|
|
osd2.wait_for_unit("network.target")
|
|
|
|
# Bootstrap ceph-mon daemon
|
|
monA.succeed(
|
|
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
|
|
"sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
|
|
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
|
|
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
|
|
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
|
|
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
|
|
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
|
|
"systemctl start ceph-mon-${cfg.monA.name}",
|
|
)
|
|
monA.wait_for_unit("ceph-mon-${cfg.monA.name}")
|
|
monA.succeed("ceph mon enable-msgr2")
|
|
monA.succeed("ceph config set mon auth_allow_insecure_global_id_reclaim false")
|
|
|
|
# Can't check ceph status until a mon is up
|
|
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
|
|
|
|
# Start the ceph-mgr daemon, it has no deps and hardly any setup
|
|
monA.succeed(
|
|
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
|
|
"sync", # to ensure shell redirection above is durable
|
|
"systemctl start ceph-mgr-${cfg.monA.name}",
|
|
)
|
|
monA.wait_for_unit("ceph-mgr-a")
|
|
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
|
|
|
|
# Send the admin keyring to the OSD machines.
|
|
monA.succeed("cp /etc/ceph/ceph.client.admin.keyring /tmp/shared")
|
|
osd0.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
|
|
osd1.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
|
|
osd2.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
|
|
|
|
# Bootstrap the BlueStore OSDs.
|
|
osd0.succeed(
|
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
|
|
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd0.name}/type",
|
|
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}/block",
|
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
|
|
'echo \'{"cephx_secret": "${cfg.osd0.key}"}\' | ceph osd new ${cfg.osd0.uuid} -i -',
|
|
)
|
|
osd1.succeed(
|
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
|
|
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd1.name}/type",
|
|
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}/block",
|
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
|
|
'echo \'{"cephx_secret": "${cfg.osd1.key}"}\' | ceph osd new ${cfg.osd1.uuid} -i -',
|
|
)
|
|
osd2.succeed(
|
|
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd2.name}",
|
|
"echo bluestore > /var/lib/ceph/osd/ceph-${cfg.osd2.name}/type",
|
|
"ln -sf /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd2.name}/block",
|
|
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd2.name}/keyring --name osd.${cfg.osd2.name} --add-key ${cfg.osd2.key}",
|
|
'echo \'{"cephx_secret": "${cfg.osd2.key}"}\' | ceph osd new ${cfg.osd2.uuid} -i -',
|
|
)
|
|
|
|
# We `sync` so that the config survives the forced crashes below.
|
|
osd0.succeed(
|
|
"ceph-osd -i ${cfg.osd0.name} --mkfs --osd-uuid ${cfg.osd0.uuid}",
|
|
"chown -R ceph:ceph /var/lib/ceph/osd",
|
|
"sync",
|
|
"systemctl start ceph-osd-${cfg.osd0.name}",
|
|
)
|
|
osd1.succeed(
|
|
"ceph-osd -i ${cfg.osd1.name} --mkfs --osd-uuid ${cfg.osd1.uuid}",
|
|
"chown -R ceph:ceph /var/lib/ceph/osd",
|
|
"sync",
|
|
"systemctl start ceph-osd-${cfg.osd1.name}",
|
|
)
|
|
osd2.succeed(
|
|
"ceph-osd -i ${cfg.osd2.name} --mkfs --osd-uuid ${cfg.osd2.uuid}",
|
|
"chown -R ceph:ceph /var/lib/ceph/osd",
|
|
"sync",
|
|
"systemctl start ceph-osd-${cfg.osd2.name}",
|
|
)
|
|
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
|
|
|
|
monA.succeed(
|
|
"ceph osd pool create multi-node-test 32 32",
|
|
"ceph osd pool ls | grep 'multi-node-test'",
|
|
|
|
# A pool that has no application associated with it stays unhealthy in
|
|
# state POOL_APP_NOT_ENABLED. Ceph only auto-associates an application
|
|
# for pools it manages itself, such as CephFS data/metadata pools
|
|
# (application "cephfs") and the pools RGW creates (application "rgw"); see
|
|
# https://docs.ceph.com/en/tentacle/rados/operations/pools/#associating-a-pool-with-an-application
|
|
# This is a plain RADOS pool, so we have to associate an application
|
|
# with it ourselves. We use the custom application name "nixos-test".
|
|
"ceph osd pool application enable multi-node-test nixos-test",
|
|
|
|
"ceph osd pool rename multi-node-test multi-node-other-test",
|
|
"ceph osd pool ls | grep 'multi-node-other-test'",
|
|
)
|
|
monA.succeed("ceph osd pool set multi-node-other-test size 2")
|
|
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
|
|
monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'")
|
|
monA.fail(
|
|
"ceph osd pool ls | grep 'multi-node-test'",
|
|
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",
|
|
)
|
|
|
|
# Shut down ceph on all machines in a very unpolite way
|
|
monA.crash()
|
|
osd0.crash()
|
|
osd1.crash()
|
|
osd2.crash()
|
|
|
|
# Start it up
|
|
osd0.start()
|
|
osd1.start()
|
|
osd2.start()
|
|
monA.start()
|
|
|
|
# Ensure the cluster comes back up again.
|
|
monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
|
|
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
|
|
|
|
# Verify the recovery.
|
|
monA.wait_until_succeeds("! ceph -s | grep -e 'unknown' -e 'pgs inactive'", timeout=60)
|
|
'';
|
|
|
|
# For `withCephfs = true`, after the regular failover, test CephFS and mounts:
|
|
# * bring up an MDS
|
|
# * create a CephFS
|
|
# * mount it via in-kernel ("kclient") and a `ceph-fuse` ("fuseclient")
|
|
# from 2 different nodes
|
|
# * verify bidirectional visibility of file changes
|
|
# * performs another crash/recovery test (without crashing the clients)
|
|
# * verifiy the mounts survive and keep working.
|
|
cephfsScript = ''
|
|
kclient.wait_for_unit("network.target")
|
|
fuseclient.wait_for_unit("network.target")
|
|
|
|
# Start the ceph-mds daemon (which provides CephFS), after creating
|
|
# its keyring and data dir.
|
|
monA.succeed(
|
|
"sudo -u ceph mkdir -p /var/lib/ceph/mds/ceph-${cfg.monA.name}/",
|
|
"ceph auth get-or-create mds.${cfg.monA.name} mon 'allow profile mds' mgr 'allow profile mds' osd 'allow rwx' mds 'allow' > /var/lib/ceph/mds/ceph-${cfg.monA.name}/keyring",
|
|
"chown ceph:ceph /var/lib/ceph/mds/ceph-${cfg.monA.name}/keyring",
|
|
"sync", # to ensure config survives the forced crashes below
|
|
"systemctl start ceph-mds-${cfg.monA.name}",
|
|
)
|
|
monA.wait_for_unit("ceph-mds-${cfg.monA.name}")
|
|
|
|
# Create a CephFS.
|
|
monA.succeed(
|
|
"ceph osd pool create cephfs-data 32 32",
|
|
"ceph osd pool create cephfs-metadata 32 32",
|
|
"ceph fs new cephfs cephfs-metadata cephfs-data",
|
|
)
|
|
# Wait for the MDS to claim the filesystem and become active.
|
|
monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
|
|
|
|
# Distribute the admin keyring (and a plain secret file for the kernel
|
|
# client) to both client machines, so that they can authenticate.
|
|
monA.succeed(
|
|
"cp /etc/ceph/ceph.client.admin.keyring /tmp/shared",
|
|
"ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > /tmp/shared/admin.secret",
|
|
)
|
|
kclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
|
|
fuseclient.succeed("cp /tmp/shared/ceph.client.admin.keyring /etc/ceph")
|
|
kclient.succeed("cp /tmp/shared/admin.secret /etc/ceph/admin.secret")
|
|
|
|
# Mount CephFS on the kernel client.
|
|
# We force the messenger v2 protocol via "ms_mode=secure"; the cluster
|
|
# has msgr2 enabled (see "ceph mon enable-msgr2" above) and the legacy v1
|
|
# protocol apparently does not reconnect reliably after the servers are restarted.
|
|
# The msgr2 monitor listens on port 3300 (instead of legacy v1 port 6789),
|
|
# so we have to point the device string at that port explicitly.
|
|
# `recover_session=clean` makes the kernel client automatically reconnect
|
|
# (discarding its stale session) after the whole cluster has been down,
|
|
# which would otherwise leave the mount blocklisted and hanging forever.
|
|
# Real CephFS use may not prefer hanging `recover_session=clean`, and
|
|
# prefer manual de-blocklisting to avoid any failed OS syscalls,
|
|
# but for this test, discarding stale sessions is good enough.
|
|
kclient.succeed("mkdir -p /mnt/cephfs")
|
|
kclient.wait_until_succeeds(
|
|
"mount -t ceph ${cfg.monA.ip}:3300:/ /mnt/cephfs -o name=admin,secretfile=/etc/ceph/admin.secret,ms_mode=secure,recover_session=clean"
|
|
)
|
|
kclient.succeed("mountpoint /mnt/cephfs")
|
|
|
|
# Mount CephFS on the FUSE client using ceph-fuse.
|
|
fuseclient.succeed("mkdir -p /mnt/cephfs")
|
|
fuseclient.wait_until_succeeds(
|
|
"ceph-fuse --id admin -m ${cfg.monA.ip}:6789 /mnt/cephfs"
|
|
)
|
|
fuseclient.succeed("mountpoint /mnt/cephfs")
|
|
|
|
# Both clients mount the same CephFS, so files written by one must be
|
|
# visible to the other. Verify this in both directions.
|
|
|
|
# Kernel client writes, FUSE client reads.
|
|
kclient.succeed("echo 'written by kclient' > /mnt/cephfs/from-kclient")
|
|
fuseclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-kclient)\" = 'written by kclient'"
|
|
)
|
|
|
|
# FUSE client writes, kernel client reads.
|
|
fuseclient.succeed("echo 'written by fuseclient' > /mnt/cephfs/from-fuseclient")
|
|
kclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-fuseclient)\" = 'written by fuseclient'"
|
|
)
|
|
|
|
# Crash test with CephFS.
|
|
# We deliberately do not crash the CephFS clients here: Their mounts must
|
|
# survive the (temporary) outage of the ceph servers and resume working
|
|
# once the cluster is healthy again.
|
|
monA.crash()
|
|
osd0.crash()
|
|
osd1.crash()
|
|
osd2.crash()
|
|
|
|
# Start it up
|
|
osd0.start()
|
|
osd1.start()
|
|
osd2.start()
|
|
monA.start()
|
|
|
|
# Ensure the cluster comes back up again.
|
|
# See the note above on why this uses `wait_until_succeeds`.
|
|
monA.wait_until_succeeds("ceph -s | grep 'mon: 1 daemons'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'quorum ${cfg.monA.name}'")
|
|
monA.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
|
|
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
|
|
|
|
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'", timeout=60)
|
|
|
|
# Ensure the MDS/CephFS comes back up again, too.
|
|
monA.wait_for_unit("ceph-mds-${cfg.monA.name}")
|
|
monA.wait_until_succeeds("ceph fs status cephfs | grep -e 'active'", timeout=60)
|
|
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
|
|
|
|
# The clients kept running across the outage, so their CephFS mounts
|
|
# should still be present and should reconnect automatically.
|
|
kclient.succeed("mountpoint /mnt/cephfs")
|
|
fuseclient.succeed("mountpoint /mnt/cephfs")
|
|
|
|
# The files written before the crash must still have the correct content.
|
|
kclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-fuseclient)\" = 'written by fuseclient'"
|
|
)
|
|
fuseclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-kclient)\" = 'written by kclient'"
|
|
)
|
|
|
|
# Ensure the mounts are still writable after recovery, in both directions.
|
|
kclient.succeed("echo 'written by kclient after recovery' > /mnt/cephfs/from-kclient-2")
|
|
fuseclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-kclient-2)\" = 'written by kclient after recovery'"
|
|
)
|
|
fuseclient.succeed("echo 'written by fuseclient after recovery' > /mnt/cephfs/from-fuseclient-2")
|
|
kclient.wait_until_succeeds(
|
|
"test \"$(cat /mnt/cephfs/from-fuseclient-2)\" = 'written by fuseclient after recovery'"
|
|
)
|
|
'';
|
|
|
|
in
|
|
{
|
|
name = "basic-multi-node-ceph-cluster-bluestore" + lib.optionalString withCephfs "-cephfs";
|
|
meta = {
|
|
maintainers = with lib.maintainers; [
|
|
nh2
|
|
benaryorg
|
|
];
|
|
};
|
|
|
|
nodes = {
|
|
monA = generateHost {
|
|
cephConfig = cephConfigMonA;
|
|
networkConfig = networkMonA;
|
|
};
|
|
osd0 = generateHost {
|
|
cephConfig = cephConfigOsd cfg.osd0;
|
|
networkConfig = networkOsd cfg.osd0;
|
|
};
|
|
osd1 = generateHost {
|
|
cephConfig = cephConfigOsd cfg.osd1;
|
|
networkConfig = networkOsd cfg.osd1;
|
|
};
|
|
osd2 = generateHost {
|
|
cephConfig = cephConfigOsd cfg.osd2;
|
|
networkConfig = networkOsd cfg.osd2;
|
|
};
|
|
}
|
|
# CephFS client machines are only needed when testing CephFS.
|
|
// lib.optionalAttrs withCephfs {
|
|
kclient = generateClientHost {
|
|
networkConfig = networkClient cfg.kclient;
|
|
};
|
|
fuseclient = generateClientHost {
|
|
networkConfig = networkClient cfg.fuseclient;
|
|
};
|
|
};
|
|
|
|
testScript =
|
|
{ ... }:
|
|
''
|
|
${helperScript}
|
|
${baseScript}
|
|
${lib.optionalString withCephfs cephfsScript}
|
|
'';
|
|
}
|