nathan/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2026-06-05 21:03:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

nixos/tests/{k3s,rke2}: merge & cleanup (#469788)

Browse Source
This commit is contained in:
rorosen
2025-12-20 14:15:44 +00:00
committed by GitHub
parent b6d3df5cd8 49bd3e5ef6
commit a14741d2ef
26 changed files with 1175 additions and 1362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/labeler.yml vendored
View File

@@ -263,7 +263,7 @@
- any-glob-to-any-file:
- nixos/modules/services/cluster/rancher/default.nix
- nixos/modules/services/cluster/rancher/k3s.nix
- nixos/tests/k3s/**/*
- nixos/tests/rancher/**/*
- pkgs/applications/networking/cluster/k3s/**/*
"6.topic: kernel":

16
nixos/tests/all-tests.nix
View File

@@ -802,7 +802,11 @@ in
jitsi-meet = runTest ./jitsi-meet.nix;
jool = import ./jool.nix { inherit pkgs runTest; };
jotta-cli = runTest ./jotta-cli.nix;
k3s = handleTest ./k3s { };
k3s = import ./rancher {
inherit pkgs runTest;
inherit (pkgs) lib;
rancherDistro = "k3s";
};
kafka = handleTest ./kafka { };
kaidan = runTest ./kaidan;
kanboard = runTest ./web-apps/kanboard.nix;
@@ -1343,7 +1347,15 @@ in
restic-rest-server = runTest ./restic-rest-server.nix;
retroarch = runTest ./retroarch.nix;
ringboard = runTest ./ringboard.nix;
rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { };
rke2 = import ./rancher {
inherit pkgs;
inherit (pkgs) lib;
runTest = runTestOn [
"aarch64-linux"
"x86_64-linux"
];
rancherDistro = "rke2";
};
rkvm = handleTest ./rkvm { };
rmfakecloud = runTest ./rmfakecloud.nix;
robustirc-bridge = runTest ./robustirc-bridge.nix;

34
nixos/tests/k3s/airgap-images.nix
View File

@@ -1,34 +0,0 @@
# A test that imports k3s airgapped images and verifies that all expected images are present
import ../make-test-python.nix (
{ lib, k3s, ... }:
{
name = "${k3s.name}-airgap-images";
meta.maintainers = lib.teams.k3s.members;
nodes.machine = _: {
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
enable = true;
role = "server";
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
];
images = [ k3s.airgap-images ];
};
};
testScript = ''
machine.wait_for_unit("k3s")
machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"")
'';
}
)

204
nixos/tests/k3s/auto-deploy-charts.nix
View File

@@ -1,204 +0,0 @@
# Tests whether container images are imported and auto deploying Helm charts,
# including the bundled traefik, work
import ../make-test-python.nix (
{
k3s,
lib,
pkgs,
...
}:
let
testImageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
paths = with pkgs; [
busybox
hello
];
};
testImage = pkgs.dockerTools.buildImage {
name = "test.local/test";
tag = "local";
# Slightly reduces the time needed to import image
compressor = "zstd";
copyToRoot = testImageEnv;
};
# pack the test helm chart as a .tgz archive
package =
pkgs.runCommand "k3s-test-chart.tgz"
{
nativeBuildInputs = [ pkgs.kubernetes-helm ];
chart = builtins.toJSON {
name = "k3s-test-chart";
version = "0.1.0";
};
values = builtins.toJSON {
restartPolicy = "Never";
runCommand = "";
image = {
repository = "foo";
tag = "1.0.0";
};
};
job = builtins.toJSON {
apiVersion = "batch/v1";
kind = "Job";
metadata = {
name = "{{ .Release.Name }}";
namespace = "{{ .Release.Namespace }}";
};
spec = {
template = {
spec = {
containers = [
{
name = "test";
image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}";
command = [ "sh" ];
args = [
"-c"
"{{ .Values.runCommand }}"
];
}
];
restartPolicy = "{{ .Values.restartPolicy }}";
};
};
};
};
passAsFile = [
"values"
"chart"
"job"
];
}
''
mkdir -p chart/templates
cp "$chartPath" chart/Chart.yaml
cp "$valuesPath" chart/values.yaml
cp "$jobPath" chart/templates/job.json
helm package chart
mv ./*.tgz $out
'';
# The common Helm chart that is used in this test
testChart = {
inherit package;
values = {
runCommand = "hello";
image = {
repository = testImage.imageName;
tag = testImage.imageTag;
};
};
};
in
{
name = "${k3s.name}-auto-deploy-helm";
meta.maintainers = lib.teams.k3s.members;
nodes.machine =
{ pkgs, ... }:
{
# k3s uses enough resources the default vm fails.
virtualisation = {
memorySize = 1536;
diskSize = 4096;
};
environment.systemPackages = [ pkgs.yq-go ];
services.k3s = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
];
images = [
# Provides the k3s Helm controller
k3s.airgap-images
testImage
];
autoDeployCharts = {
# regular test chart that should get installed
hello = testChart;
# disabled chart that should not get installed
disabled = testChart // {
enable = false;
};
# chart with values set via YAML file
values-file = testChart // {
# Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
# https://github.com/NixOS/nix/issues/12407
values =
/.
+ builtins.unsafeDiscardStringContext (
builtins.toFile "k3s-test-chart-values.yaml" ''
runCommand: "echo 'Hello, file!'"
image:
repository: test.local/test
tag: local
''
);
};
# advanced chart that should get installed in the "test" namespace with a custom
# timeout and overridden values
advanced = testChart // {
# create the "test" namespace via extraDeploy for testing
extraDeploy = [
{
apiVersion = "v1";
kind = "Namespace";
metadata.name = "test";
}
];
extraFieldDefinitions = {
spec = {
# overwrite chart values
valuesContent = ''
runCommand: "echo 'advanced hello'"
image:
repository: ${testImage.imageName}
tag: ${testImage.imageTag}
'';
# overwrite the chart namespace
targetNamespace = "test";
# set a custom timeout
timeout = "69s";
};
};
};
};
};
};
testScript = # python
''
import json
machine.wait_for_unit("k3s")
# check existence/absence of chart manifest files
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
# check that the timeout is set correctly, select only the first doc in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
# wait for test jobs to complete
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
# check output of test jobs
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
# strip the output to remove trailing whitespaces
t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
# wait for bundled traefik deployment
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
'';
}
)

124
nixos/tests/k3s/auto-deploy.nix
View File

@@ -1,124 +0,0 @@
# Tests whether container images are imported and auto deploying manifests work
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
pauseImageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = pauseImageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
in
{
name = "${k3s.name}-auto-deploy";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ k3s ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.package = k3s;
# Slightly reduce resource usage
services.k3s.extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
];
services.k3s.images = [
pauseImage
helloImage
];
services.k3s.manifests = {
absent = {
enable = false;
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "absent";
};
};
present = {
target = "foo-namespace.yaml";
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "foo";
};
};
hello.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "hello";
spec = {
template.spec = {
containers = [
{
name = "hello";
image = "test.local/hello:local";
}
];
restartPolicy = "OnFailure";
};
};
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("k3s")
# check existence of the manifest files
machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")
# check if container images got imported
machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'")
# check if resources of manifests got created
machine.wait_until_succeeds("kubectl get ns foo")
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
machine.fail("kubectl get ns absent")
'';
meta.maintainers = lib.teams.k3s.members;
}
)

59
nixos/tests/k3s/containerd-config.nix
View File

@@ -1,59 +0,0 @@
# A test that containerdConfigTemplate settings get written to containerd/config.toml
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
nodeName = "test";
in
{
name = "${k3s.name}-containerd-config";
nodes.machine =
{ ... }:
{
environment.systemPackages = [ pkgs.jq ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-name ${nodeName}"
];
containerdConfigTemplate = ''
# Base K3s config
{{ template "base" . }}
# MAGIC COMMENT
'';
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("k3s")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the config template file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl")
t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment")
# test whether the config file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml")
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
'';
meta.maintainers = lib.teams.k3s.members;
}
)

34
nixos/tests/k3s/default.nix
View File

@@ -1,34 +0,0 @@
{
system ? builtins.currentSystem,
pkgs ? import ../../.. { inherit system; },
lib ? pkgs.lib,
}:
let
allK3s = lib.filterAttrs (
n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success
) pkgs;
in
{
airgap-images = lib.mapAttrs (
_: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
) allK3s;
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
auto-deploy-charts = lib.mapAttrs (
_: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
) allK3s;
containerd-config = lib.mapAttrs (
_: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
) allK3s;
etcd = lib.mapAttrs (
_: k3s:
import ./etcd.nix {
inherit system pkgs k3s;
inherit (pkgs) etcd;
}
) allK3s;
kubelet-config = lib.mapAttrs (
_: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; }
) allK3s;
multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
}

126
nixos/tests/k3s/etcd.nix
View File

@@ -1,126 +0,0 @@
# Tests K3s with Etcd backend
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
etcd,
...
}:
{
name = "${k3s.name}-etcd";
nodes = {
etcd =
{ ... }:
{
services.etcd = {
enable = true;
openFirewall = true;
listenClientUrls = [
"http://192.168.1.1:2379"
"http://127.0.0.1:2379"
];
listenPeerUrls = [ "http://192.168.1.1:2380" ];
initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
initialCluster = [ "etcd=http://192.168.1.1:2380" ];
};
networking = {
useDHCP = false;
defaultGateway = "192.168.1.1";
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
{
address = "192.168.1.1";
prefixLength = 24;
}
];
};
};
k3s =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [ jq ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
enable = true;
role = "server";
package = k3s;
extraFlags = [
"--datastore-endpoint=\"http://192.168.1.1:2379\""
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-ip 192.168.1.2"
];
};
networking = {
firewall = {
allowedTCPPorts = [
2379
2380
6443
];
allowedUDPPorts = [ 8472 ];
};
useDHCP = false;
defaultGateway = "192.168.1.2";
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
{
address = "192.168.1.2";
prefixLength = 24;
}
];
};
};
};
testScript = # python
''
with subtest("should start etcd"):
etcd.start()
etcd.wait_for_unit("etcd.service")
with subtest("should wait for etcdctl endpoint status to succeed"):
etcd.wait_until_succeeds("etcdctl endpoint status")
with subtest("should wait for etcdctl endpoint health to succeed"):
etcd.wait_until_succeeds("etcdctl endpoint health")
with subtest("should start k3s"):
k3s.start()
k3s.wait_for_unit("k3s")
with subtest("should test if kubectl works"):
k3s.wait_until_succeeds("k3s kubectl get node")
with subtest("should wait for service account to show up; takes a sec"):
k3s.wait_until_succeeds("k3s kubectl get serviceaccount default")
with subtest("should create a sample secret object"):
k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
with subtest("should check if secret is correct"):
k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
with subtest("should have a secret in database"):
etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
with subtest("should delete the secret"):
k3s.succeed("k3s kubectl delete secret nixossecret")
with subtest("should not have a secret in database"):
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
'';
meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members;
}
)

76
nixos/tests/k3s/kubelet-config.nix
View File

@@ -1,76 +0,0 @@
# A test that sets extra kubelet configuration and enables graceful node shutdown
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
nodeName = "test";
shutdownGracePeriod = "1m13s";
shutdownGracePeriodCriticalPods = "13s";
podsPerCore = 3;
memoryThrottlingFactor = 0.69;
containerLogMaxSize = "5Mi";
in
{
name = "${k3s.name}-kubelet-config";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-name ${nodeName}"
];
gracefulNodeShutdown = {
enable = true;
inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
};
extraKubeletConfig = {
inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
};
};
};
testScript = # python
''
import json
start_all()
machine.wait_for_unit("k3s")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the kubelet registered an inhibitor lock
machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"")
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
# get the kubeletconfig
kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
with subtest("Kubelet config values are set correctly"):
t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}")
t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}")
t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore})
t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor})
t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
'';
meta.maintainers = lib.teams.k3s.members;
}
)

200
nixos/tests/k3s/multi-node.nix
View File

@@ -1,200 +0,0 @@
# A test that runs a multi-node k3s cluster and verify pod networking works across nodes
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
paths = with pkgs; [
tini
bashInteractive
coreutils
socat
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = imageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
# A daemonset that responds 'server' on port 8000
networkTestDaemonset = pkgs.writeText "test.yml" ''
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: test
labels:
name: test
spec:
selector:
matchLabels:
name: test
template:
metadata:
labels:
name: test
spec:
containers:
- name: test
image: test.local/pause:local
imagePullPolicy: Never
resources:
limits:
memory: 20Mi
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
'';
tokenFile = pkgs.writeText "token" "p@s$w0rd";
in
{
name = "${k3s.name}-multi-node";
nodes = {
server =
{ nodes, pkgs, ... }:
{
environment.systemPackages = with pkgs; [
gzip
jq
];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
inherit tokenFile;
enable = true;
role = "server";
package = k3s;
images = [ pauseImage ];
clusterInit = true;
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
"--node-ip ${nodes.server.networking.primaryIPAddress}"
# The interface selection logic of flannel would normally use eth0, as the nixos
# testing driver sets a default route via dev eth0. However, in test setups we
# have to use eth1 for inter-node communication.
"--flannel-iface eth1"
];
};
networking.firewall.allowedTCPPorts = [
2379
2380
6443
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
server2 =
{ nodes, pkgs, ... }:
{
environment.systemPackages = with pkgs; [
gzip
jq
];
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
inherit tokenFile;
enable = true;
package = k3s;
images = [ pauseImage ];
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
clusterInit = false;
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
"--node-ip ${nodes.server2.networking.primaryIPAddress}"
"--flannel-iface eth1"
];
};
networking.firewall.allowedTCPPorts = [
2379
2380
6443
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
agent =
{ nodes, pkgs, ... }:
{
virtualisation.memorySize = 1024;
virtualisation.diskSize = 2048;
services.k3s = {
inherit tokenFile;
enable = true;
role = "agent";
package = k3s;
images = [ pauseImage ];
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
extraFlags = [
"--pause-image test.local/pause:local"
"--node-ip ${nodes.agent.networking.primaryIPAddress}"
"--flannel-iface eth1"
];
};
networking.firewall.allowedTCPPorts = [ 6443 ];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
};
testScript = # python
''
start_all()
machines = [server, server2, agent]
for m in machines:
m.wait_for_unit("k3s")
# wait for the agent to show up
server.wait_until_succeeds("k3s kubectl get node agent")
for m in machines:
m.succeed("k3s check-config")
server.succeed("k3s kubectl cluster-info")
# Also wait for our service account to show up; it takes a sec
server.wait_until_succeeds("k3s kubectl get serviceaccount default")
# Now create a pod on each node via a daemonset and verify they can talk to each other.
server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
# Get pod IPs
pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
# Verify each server can ping each pod ip
for pod_ip in pod_ips:
server.succeed(f"ping -c 1 {pod_ip}")
server2.succeed(f"ping -c 1 {pod_ip}")
agent.succeed(f"ping -c 1 {pod_ip}")
# Verify the pods can talk to each other
for pod in pods:
resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
t.assertEqual(resp.strip(), "server")
'';
meta.maintainers = lib.teams.k3s.members;
}
)

115
nixos/tests/k3s/single-node.nix
View File

@@ -1,115 +0,0 @@
# A test that runs a single node k3s cluster and verify a pod can run
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.streamLayeredImage {
name = "test.local/pause";
tag = "local";
contents = imageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
testPodYaml = pkgs.writeText "test.yml" ''
apiVersion: v1
kind: Pod
metadata:
name: test
spec:
containers:
- name: test
image: test.local/pause:local
imagePullPolicy: Never
command: ["sh", "-c", "sleep inf"]
'';
in
{
name = "${k3s.name}-single-node";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
k3s
gzip
];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.package = k3s;
# Slightly reduce resource usage
services.k3s.extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
];
users.users = {
noprivs = {
isNormalUser = true;
description = "Can't access k3s by default";
password = "*";
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("k3s")
machine.succeed("kubectl cluster-info")
machine.fail("sudo -u noprivs kubectl cluster-info")
machine.succeed("k3s check-config")
machine.succeed(
"${pauseImage} | ctr image import -"
)
# Also wait for our service account to show up; it takes a sec
machine.wait_until_succeeds("kubectl get serviceaccount default")
machine.succeed("kubectl apply -f ${testPodYaml}")
machine.succeed("kubectl wait --for 'condition=Ready' pod/test")
machine.succeed("kubectl delete -f ${testPodYaml}")
# regression test for #176445
machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'")
with subtest("Run k3s-killall"):
# Call the killall script with a clean path to assert that
# all required commands are wrapped
output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr")
t.assertNotIn("command not found", output, "killall script contains unknown command")
# Check that killall cleaned up properly
machine.fail("systemctl is-active k3s.service")
machine.fail("systemctl list-units | grep containerd")
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
machine.fail("ip netns show | grep cni-")
'';
meta.maintainers = lib.teams.k3s.members;
}
)

40
nixos/tests/rancher/airgap-images.nix Normal file
View File

@@ -0,0 +1,40 @@
# A test that imports k3s airgapped images and verifies that all expected images are present
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
{
name = "${rancherPackage.name}-airgap-images";
nodes.machine = _: {
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images =
coreImages
++ {
k3s = [ rancherPackage.airgap-images ];
rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages
}
.${rancherDistro};
};
};
testScript = ''
machine.wait_for_unit("${serviceName}")
machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

230
nixos/tests/rancher/auto-deploy-charts.nix Normal file
View File

@@ -0,0 +1,230 @@
# Tests whether container images are imported and auto deploying Helm charts,
# including the bundled traefik or ingress-nginx, work
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
testImageEnv = pkgs.buildEnv {
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
busybox
hello
];
};
testImage = pkgs.dockerTools.buildImage {
name = "test.local/test";
tag = "local";
# Slightly reduces the time needed to import image
compressor = "zstd";
copyToRoot = testImageEnv;
};
# pack the test helm chart as a .tgz archive
package =
pkgs.runCommand "${rancherDistro}-test-chart.tgz"
{
nativeBuildInputs = [ pkgs.kubernetes-helm ];
chart = builtins.toJSON {
name = "${rancherDistro}-test-chart";
version = "0.1.0";
};
values = builtins.toJSON {
restartPolicy = "Never";
runCommand = "";
image = {
repository = "foo";
tag = "1.0.0";
};
};
job = builtins.toJSON {
apiVersion = "batch/v1";
kind = "Job";
metadata = {
name = "{{ .Release.Name }}";
namespace = "{{ .Release.Namespace }}";
};
spec = {
template = {
spec = {
containers = [
{
name = "test";
image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}";
command = [ "sh" ];
args = [
"-c"
"{{ .Values.runCommand }}"
];
}
];
restartPolicy = "{{ .Values.restartPolicy }}";
};
};
};
};
passAsFile = [
"values"
"chart"
"job"
];
}
''
mkdir -p chart/templates
cp "$chartPath" chart/Chart.yaml
cp "$valuesPath" chart/values.yaml
cp "$jobPath" chart/templates/job.json
helm package chart
mv ./*.tgz $out
'';
# The common Helm chart that is used in this test
testChart = {
inherit package;
values = {
runCommand = "hello";
image = {
repository = testImage.imageName;
tag = testImage.imageTag;
};
};
};
in
{
name = "${rancherPackage.name}-auto-deploy-helm";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
kubectl
yq-go
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
disable =
{
k3s = lib.remove "traefik" disabledComponents;
rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;
}
.${rancherDistro};
images =
coreImages
# Provides the k3s Helm controller
++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images
++ [
testImage
];
autoDeployCharts = {
# regular test chart that should get installed
hello = testChart;
# disabled chart that should not get installed
disabled = testChart // {
enable = false;
};
# chart with values set via YAML file
values-file = testChart // {
# Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
# https://github.com/NixOS/nix/issues/12407
values =
/.
+ builtins.unsafeDiscardStringContext (
builtins.toFile "${rancherDistro}-test-chart-values.yaml" ''
runCommand: "echo 'Hello, file!'"
image:
repository: test.local/test
tag: local
''
);
};
# advanced chart that should get installed in the "test" namespace with a custom
# timeout and overridden values
advanced = testChart // {
# create the "test" namespace via extraDeploy for testing
extraDeploy = [
{
apiVersion = "v1";
kind = "Namespace";
metadata.name = "test";
}
];
extraFieldDefinitions = {
spec = {
# overwrite chart values
valuesContent = ''
runCommand: "echo 'advanced hello'"
image:
repository: ${testImage.imageName}
tag: ${testImage.imageTag}
'';
# overwrite the chart namespace
targetNamespace = "test";
# set a custom timeout
timeout = "69s";
};
};
};
};
};
};
testScript = # python
let
manifestFormat =
{
k3s = "yaml";
rke2 = "json";
}
.${rancherDistro};
in
''
import json
machine.wait_for_unit("${serviceName}")
# check existence/absence of chart manifest files
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")
# check that the timeout is set correctly, select only the first item in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}"))
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
# wait for test jobs to complete
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
# check output of test jobs
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
# strip the output to remove trailing whitespaces
t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
# wait for bundled ingress deployment
${
{
k3s = ''
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
'';
rke2 = ''
machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180)
'';
}
.${rancherDistro}
}
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

134
nixos/tests/rancher/auto-deploy.nix Normal file
View File

@@ -0,0 +1,134 @@
# Tests whether container images are imported and auto deploying manifests work
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
pauseImageEnv = pkgs.buildEnv {
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = pauseImageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
manifestFormat =
{
k3s = "yaml";
rke2 = "json";
}
.${rancherDistro};
in
{
name = "${rancherPackage.name}-auto-deploy";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
kubectl
cri-tools
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
];
images = coreImages ++ [
pauseImage
helloImage
];
manifests = {
absent = {
enable = false;
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "absent";
};
};
present = {
target = "foo-namespace.${manifestFormat}";
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "foo";
};
};
hello.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "hello";
spec = {
template.spec = {
containers = [
{
name = "hello";
image = "test.local/hello:local";
}
];
restartPolicy = "OnFailure";
};
};
};
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("${serviceName}")
# check existence of the manifest files
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
# check if container images got imported
# for some reason, RKE2 also uses /run/k3s
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'")
# check if resources of manifests got created
machine.wait_until_succeeds("kubectl get ns foo")
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
machine.fail("kubectl get ns absent")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

59
nixos/tests/rancher/containerd-config.nix Normal file
View File

@@ -0,0 +1,59 @@
# A test that containerdConfigTemplate settings get written to containerd/config.toml
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
nodeName = "test";
in
{
name = "${rancherPackage.name}-containerd-config";
nodes.machine =
{ ... }:
{
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
inherit nodeName;
containerdConfigTemplate = ''
# Base ${rancherDistro} config
{{ template "base" . }}
# MAGIC COMMENT
'';
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("${serviceName}")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the config template file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl")
t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment")
# test whether the config file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml")
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

116
nixos/tests/rancher/default.nix Normal file
View File

@@ -0,0 +1,116 @@
{
runTest,
pkgs,
lib,
# service/package name to test
rancherDistro,
...
}:
let
allPackages = lib.filterAttrs (
name: package:
builtins.match "^${rancherDistro}(_[[:digit:]]+)+$" name != null
&& (builtins.tryEval package).success
) pkgs;
allTests =
let
mkTestArgs = rancherPackage: {
inherit rancherDistro rancherPackage;
# systemd service name
serviceName =
{
k3s = "k3s";
rke2 = "rke2-server";
}
.${rancherDistro};
# list passed to services.*.disable,
# for slightly reduced resource usage
disabledComponents =
{
k3s = [
"coredns"
"local-storage"
"metrics-server"
"servicelb"
"traefik"
];
rke2 = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
}
.${rancherDistro};
# images that must be present for all tests
coreImages =
{
k3s = [ ];
rke2 =
{
aarch64-linux = [
rancherPackage.images-core-linux-arm64-tar-zst
rancherPackage.images-canal-linux-arm64-tar-zst
];
x86_64-linux = [
rancherPackage.images-core-linux-amd64-tar-zst
rancherPackage.images-canal-linux-amd64-tar-zst
];
}
.${pkgs.stdenv.hostPlatform.system}
or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}");
}
.${rancherDistro};
# virtualization.* attrs, since all distros
# need more resources than the default
vmResources =
{
k3s = {
memorySize = 1536;
diskSize = 4096;
};
rke2 = {
cores = 4;
memorySize = 4096;
diskSize = 8092;
};
}
.${rancherDistro};
};
mkTests =
path:
lib.mapAttrs (
name: package:
runTest {
imports = [ path ];
_module.args = mkTestArgs package;
}
) allPackages;
in
{
airgap-images = mkTests ./airgap-images.nix;
auto-deploy = mkTests ./auto-deploy.nix;
auto-deploy-charts = mkTests ./auto-deploy-charts.nix;
containerd-config = mkTests ./containerd-config.nix;
etcd = mkTests ./etcd.nix;
kubelet-config = mkTests ./kubelet-config.nix;
multi-node = mkTests ./multi-node.nix;
single-node = mkTests ./single-node.nix;
};
in
allTests
// {
all = lib.concatMapAttrs (
testType: lib.mapAttrs' (package: lib.nameValuePair "${testType}-${package}")
) allTests;
}

129
nixos/tests/rancher/etcd.nix Normal file
View File

@@ -0,0 +1,129 @@
# Tests K3s with Etcd backend
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
{
name = "${rancherPackage.name}-etcd";
nodes = {
etcd =
{ ... }:
{
services.etcd = {
enable = true;
openFirewall = true;
listenClientUrls = [
"http://192.168.1.1:2379"
"http://127.0.0.1:2379"
];
listenPeerUrls = [ "http://192.168.1.1:2380" ];
initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
initialCluster = [ "etcd=http://192.168.1.1:2380" ];
};
networking = {
useDHCP = false;
defaultGateway = "192.168.1.1";
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
{
address = "192.168.1.1";
prefixLength = 24;
}
];
};
};
server =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
nodeIP = "192.168.1.2";
extraFlags = [
"--datastore-endpoint=\"http://192.168.1.1:2379\""
];
};
networking = {
firewall = {
allowedTCPPorts = [
2379
2380
6443
];
allowedUDPPorts = [ 8472 ];
};
useDHCP = false;
defaultGateway = "192.168.1.2";
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
{
address = "192.168.1.2";
prefixLength = 24;
}
];
};
};
};
testScript = # python
''
with subtest("should start etcd"):
etcd.start()
etcd.wait_for_unit("etcd.service")
with subtest("should wait for etcdctl endpoint status to succeed"):
etcd.wait_until_succeeds("etcdctl endpoint status")
with subtest("should wait for etcdctl endpoint health to succeed"):
etcd.wait_until_succeeds("etcdctl endpoint health")
with subtest("should start ${rancherDistro}"):
server.start()
server.wait_for_unit("${serviceName}")
with subtest("should test if kubectl works"):
server.wait_until_succeeds("kubectl get node")
with subtest("should wait for service account to show up; takes a sec"):
server.wait_until_succeeds("kubectl get serviceaccount default")
with subtest("should create a sample secret object"):
server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
with subtest("should check if secret is correct"):
server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
with subtest("should have a secret in database"):
etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
with subtest("should delete the secret"):
server.succeed("kubectl delete secret nixossecret")
with subtest("should not have a secret in database"):
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
'';
meta.maintainers =
pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

75
nixos/tests/rancher/kubelet-config.nix Normal file
View File

@@ -0,0 +1,75 @@
# A test that sets extra kubelet configuration and enables graceful node shutdown
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
nodeName = "test";
shutdownGracePeriod = "1m13s";
shutdownGracePeriodCriticalPods = "13s";
podsPerCore = 3;
memoryThrottlingFactor = 0.69;
containerLogMaxSize = "5Mi";
in
{
name = "${rancherPackage.name}-kubelet-config";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
inherit nodeName;
gracefulNodeShutdown = {
enable = true;
inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
};
extraKubeletConfig = {
inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
};
};
};
testScript = # python
''
import json
start_all()
machine.wait_for_unit("${serviceName}")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the kubelet registered an inhibitor lock
machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"")
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
# get the kubeletconfig
kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
with subtest("Kubelet config values are set correctly"):
t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}")
t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}")
t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore})
t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor})
t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

250
nixos/tests/rancher/multi-node.nix Normal file
View File

@@ -0,0 +1,250 @@
# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
bashInteractive
coreutils
socat
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = imageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
# A daemonset that responds 'server' on port 8000
networkTestDaemonset = pkgs.writeText "test.yml" ''
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: test
labels:
name: test
spec:
selector:
matchLabels:
name: test
template:
metadata:
labels:
name: test
spec:
containers:
- name: test
image: test.local/pause:local
imagePullPolicy: Never
resources:
limits:
memory: 20Mi
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
'';
tokenFile = pkgs.writeText "token" "p@s$w0rd";
supervisorPort =
{
k3s = "6443";
rke2 = "9345";
}
.${rancherDistro};
in
{
name = "${rancherPackage.name}-multi-node";
nodes = {
server =
{
nodes,
pkgs,
config,
...
}:
{
environment.systemPackages = with pkgs; [
kubectl
gzip
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = lib.mkMerge [
{
inherit tokenFile;
enable = true;
role = "server";
package = rancherPackage;
images = coreImages ++ [ pauseImage ];
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
];
}
{
k3s = {
clusterInit = true;
extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition
};
# The interface selection logic of flannel & canal would normally use eth0, as
# the nixos testing driver sets a default route via dev eth0. However, in test
# setups we have to use eth1 for inter-node communication.
# For K3s this can be handled via --flannel-iface, but RKE2's canal has to be
# configured with this manifest.
rke2.manifests.canal-config.content = {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChartConfig";
metadata = {
name = "rke2-canal";
namespace = "kube-system";
};
# spec.valuesContent needs to a string, either json or yaml
spec.valuesContent = builtins.toJSON {
flannel.iface = "eth1";
};
};
}
.${rancherDistro}
];
networking.firewall.enable = false;
networking.firewall.allowedTCPPorts = [
2379
2380
6443
]
++ lib.optionals (rancherDistro == "rke2") [
9099
9345
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
server2 =
{
nodes,
pkgs,
config,
...
}:
{
virtualisation = vmResources;
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "server";
package = rancherPackage;
images = coreImages ++ [ pauseImage ];
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}";
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
]
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
};
networking.firewall.enable = false;
networking.firewall.allowedTCPPorts = [
2379
2380
6443
]
++ lib.optionals (rancherDistro == "rke2") [
9099
9345
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
agent =
{
nodes,
pkgs,
config,
...
}:
{
virtualisation = vmResources;
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "agent";
package = rancherPackage;
images = coreImages ++ [ pauseImage ];
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}";
nodeIP = config.networking.primaryIPAddress;
extraFlags = [
"--pause-image test.local/pause:local"
]
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
};
networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099;
networking.firewall.allowedUDPPorts = [ 8472 ];
};
};
testScript = # python
''
start_all()
servers = [server, server2]
for m in servers:
m.wait_for_unit("${serviceName}")
# wait for the agent to show up
server.wait_until_succeeds("kubectl get node agent")
${lib.optionalString (rancherDistro == "k3s") ''
for m in machines:
m.succeed("k3s check-config")
''}
server.succeed("kubectl cluster-info")
# Also wait for our service account to show up; it takes a sec
server.wait_until_succeeds("kubectl get serviceaccount default")
# Now create a pod on each node via a daemonset and verify they can talk to each other.
server.succeed("kubectl apply -f ${networkTestDaemonset}")
server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
# Get pod IPs
pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
# Verify each server can ping each pod ip
for pod_ip in pod_ips:
server.succeed(f"ping -c 1 {pod_ip}")
server2.succeed(f"ping -c 1 {pod_ip}")
agent.succeed(f"ping -c 1 {pod_ip}")
# Verify the pods can talk to each other
for pod in pods:
resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
t.assertEqual(resp.strip(), "server")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

114
nixos/tests/rancher/single-node.nix Normal file
View File

@@ -0,0 +1,114 @@
# A test that runs a single node rancher cluster and verifies a pod can run
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.buildLayeredImage {
name = "test.local/pause";
tag = "local";
contents = imageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
testPodYaml = pkgs.writeText "test.yaml" ''
apiVersion: v1
kind: Pod
metadata:
name: test
spec:
containers:
- name: test
image: test.local/pause:local
imagePullPolicy: Never
command: ["sh", "-c", "sleep inf"]
'';
in
{
name = "${rancherPackage.name}-single-node";
nodes.machine =
{ config, pkgs, ... }:
{
environment.systemPackages = with pkgs; [
kubectl
gzip
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images = coreImages ++ [ pauseImage ];
extraFlags = [
"--pause-image test.local/pause:local"
];
};
users.users = {
noprivs = {
isNormalUser = true;
description = "Can't access ${rancherDistro} by default";
password = "*";
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("${serviceName}")
machine.succeed("kubectl cluster-info")
machine.fail("sudo -u noprivs kubectl cluster-info")
${lib.optionalString (rancherDistro == "k3s") ''
machine.succeed("k3s check-config")
''}
# Also wait for our service account to show up; it takes a sec
machine.wait_until_succeeds("kubectl get serviceaccount default")
machine.succeed("kubectl apply -f ${testPodYaml}")
machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s")
machine.succeed("kubectl delete -f ${testPodYaml}")
# regression test for #176445
machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'")
with subtest("Run ${rancherDistro}-killall"):
# Call the killall script with a clean path to assert that
# all required commands are wrapped
output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr")
t.assertNotIn("command not found", output, "killall script contains unknown command")
# Check that killall cleaned up properly
machine.fail("systemctl is-active ${serviceName}.service")
machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5)
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
machine.fail("ip netns show | grep cni-")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}

14
nixos/tests/rke2/default.nix
View File

@@ -1,14 +0,0 @@
{
system ? builtins.currentSystem,
pkgs ? import ../../.. { inherit system; },
lib ? pkgs.lib,
}:
let
allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs;
in
{
# Run a single node rke2 cluster and verify a pod can run
singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2;
# Run a multi-node rke2 cluster and verify pod networking works across nodes
multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2;
}

207
nixos/tests/rke2/multi-node.nix
View File

@@ -1,207 +0,0 @@
import ../make-test-python.nix (
{
pkgs,
lib,
rke2,
...
}:
let
throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
coreImages =
{
aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
canalImages =
{
aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
compressor = "zstd";
copyToRoot = pkgs.buildEnv {
name = "rke2-hello-image-env";
paths = with pkgs; [
coreutils
socat
];
};
};
tokenFile = pkgs.writeText "token" "p@s$w0rd";
agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";
# Let flannel use eth1 to enable inter-node communication in tests
canalConfig = {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChartConfig";
metadata = {
name = "rke2-canal";
namespace = "kube-system";
};
# spec.valuesContent needs to a string, either json or yaml
spec.valuesContent = builtins.toJSON {
flannel.iface = "eth1";
};
};
in
{
name = "${rke2.name}-multi-node";
meta.maintainers = rke2.meta.maintainers;
nodes = {
server =
{
config,
nodes,
pkgs,
...
}:
{
# Canal CNI with VXLAN
networking.firewall.allowedUDPPorts = [ 8472 ];
networking.firewall.allowedTCPPorts = [
# Kubernetes API
6443
# Canal CNI health checks
9099
# RKE2 supervisor API
9345
];
# RKE2 needs more resources than the default
virtualisation.cores = 4;
virtualisation.memorySize = 4096;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "server";
package = rke2;
inherit tokenFile;
inherit agentTokenFile;
# Without nodeIP the apiserver starts with the wrong service IP family
nodeIP = config.networking.primaryIPAddress;
disable = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
canal-config.content = canalConfig;
# A daemonset that responds 'hello' on port 8000
network-test.content = {
apiVersion = "apps/v1";
kind = "DaemonSet";
metadata = {
name = "test";
labels.name = "test";
};
spec = {
selector.matchLabels.name = "test";
template = {
metadata.labels.name = "test";
spec.containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
imagePullPolicy = "Never";
command = [
"socat"
"TCP4-LISTEN:8000,fork"
"EXEC:echo hello"
];
}
];
};
};
};
};
};
};
agent =
{
config,
nodes,
pkgs,
...
}:
{
# Canal CNI health checks
networking.firewall.allowedTCPPorts = [ 9099 ];
# Canal CNI with VXLAN
networking.firewall.allowedUDPPorts = [ 8472 ];
# The agent node can work with less resources
virtualisation.memorySize = 2048;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "agent";
package = rke2;
tokenFile = agentTokenFile;
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";
nodeIP = config.networking.primaryIPAddress;
manifests.canal-config.content = canalConfig;
images = [
coreImages
canalImages
helloImage
];
};
};
};
testScript =
let
kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
jq = "${pkgs.jq}/bin/jq";
in
# python
''
start_all()
server.wait_for_unit("rke2-server")
agent.wait_for_unit("rke2-agent")
# Wait for the agent to be ready
server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""")
server.succeed("${kubectl} cluster-info")
server.wait_until_succeeds("${kubectl} get serviceaccount default")
# Now verify that each daemonset pod can talk to each other.
server.wait_until_succeeds(
f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'
)
# Get pod IPs
pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()
pod_ips = [
server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods
]
# Verify each node can ping each pod ip
for pod_ip in pod_ips:
# The CNI sometimes needs a little time
server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
# Verify the server can exec into the pod
for pod in pods:
resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip()
assert resp == "hello", f"Unexpected response from hello daemonset: {resp}"
'';
}
)

144
nixos/tests/rke2/single-node.nix
View File

@@ -1,144 +0,0 @@
import ../make-test-python.nix (
{
pkgs,
lib,
rke2,
...
}:
let
throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
coreImages =
{
aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
canalImages =
{
aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
compressor = "zstd";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
# A ConfigMap in regular yaml format
cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "from-file";
data.username = "foo-file";
};
in
{
name = "${rke2.name}-single-node";
meta.maintainers = rke2.meta.maintainers;
nodes.machine =
{
config,
nodes,
pkgs,
...
}:
{
# RKE2 needs more resources than the default
virtualisation.cores = 4;
virtualisation.memorySize = 4096;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "server";
package = rke2;
# Without nodeIP the apiserver starts with the wrong service IP family
nodeIP = config.networking.primaryIPAddress;
# Slightly reduce resource consumption
disable = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
test-job.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "test";
spec.template.spec = {
containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
}
];
restartPolicy = "Never";
};
};
disabled = {
enable = false;
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "disabled";
data.username = "foo";
};
};
from-file.source = "${cmFile}";
custom-target = {
enable = true;
target = "my-manifest.json";
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "custom-target";
data.username = "foo-custom";
};
};
};
};
};
testScript =
let
kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
in
# python
''
start_all()
with subtest("Start cluster"):
machine.wait_for_unit("rke2-server")
machine.succeed("${kubectl} cluster-info")
machine.wait_until_succeeds("${kubectl} get serviceaccount default")
with subtest("Test job completes successfully"):
machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test")
output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip()
assert output == "Hello, world!", f"unexpected output of test job: {output}"
with subtest("ConfigMap from-file exists"):
output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}"
with subtest("ConfigMap custom-target exists"):
# Check that the file exists at the custom target path
machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json")
output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}"
with subtest("Disabled ConfigMap doesn't exist"):
machine.fail("${kubectl} get cm disabled")
'';
}
)

11
pkgs/applications/networking/cluster/k3s/builder.nix
View File

@@ -470,14 +470,11 @@ buildGoModule (finalAttrs: {
;
tests =
let
mkTests =
version:
let
k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version);
in
lib.mapAttrs (name: value: nixosTests.k3s.${name}.${k3s_version}) nixosTests.k3s;
versionedPackage = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor k3sVersion);
in
mkTests k3sVersion;
lib.mapAttrs (name: _: nixosTests.k3s.${name}.${versionedPackage}) (
lib.filterAttrs (n: _: n != "all") nixosTests.k3s
);
imagesList = throw "k3s.imagesList was removed";
airgapImages = throw "k3s.airgapImages was renamed to k3s.airgap-images";
airgapImagesAmd64 = throw "k3s.airgapImagesAmd64 was renamed to k3s.airgap-images-amd64-tar-zst";

2
pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md
View File

@@ -7,7 +7,7 @@ A K3s maintainer, maintains K3s's:
- [documentation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/README.md)
- [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s)
- [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22)
- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s)
- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/rancher)
- [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher)
- [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating)
- updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/)

22
pkgs/applications/networking/cluster/rke2/builder.nix
View File

@@ -22,6 +22,7 @@ lib:
makeWrapper,
fetchzip,
fetchurl,
versionCheckHook,
# Runtime dependencies
procps,
@@ -42,7 +43,6 @@ lib:
# Testing dependencies
nixosTests,
testers,
}:
buildGoModule (finalAttrs: {
pname = "rke2";
@@ -129,25 +129,19 @@ buildGoModule (finalAttrs: {
go tool nm $out/bin/.rke2-wrapped | grep '_Cfunc__goboringcrypto_' > /dev/null
runHook postInstallCheck
'';
nativeInstallCheckInputs = [ versionCheckHook ];
versionCheckProgramArg = "--version";
passthru = {
inherit updateScript;
tests =
let
moduleTests =
let
package_version =
"rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version);
in
lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2;
versionedPackage =
"rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version);
in
{
version = testers.testVersion {
package = finalAttrs.finalPackage;
version = "v${finalAttrs.version}";
};
}
// moduleTests;
lib.mapAttrs (name: _: nixosTests.rke2.${name}.${versionedPackage}) (
lib.filterAttrs (n: _: n != "all") nixosTests.rke2
);
}
// (lib.mapAttrs (_: value: fetchurl value) imagesVersions);