mirror of
https://github.com/NixOS/nixpkgs.git
synced 2026-06-05 21:03:40 +00:00
nixos/tests/{k3s,rke2}: migrate to runTest
Also exposed a .all attr alongside existing tests, for easily running all tests for all package versions.
This commit is contained in:
azey
@@ -800,7 +800,11 @@ in
|
||||
jitsi-meet = runTest ./jitsi-meet.nix;
|
||||
jool = import ./jool.nix { inherit pkgs runTest; };
|
||||
jotta-cli = runTest ./jotta-cli.nix;
|
||||
k3s = handleTest ./rancher { rancherDistro = "k3s"; };
|
||||
k3s = import ./rancher {
|
||||
inherit pkgs runTest;
|
||||
inherit (pkgs) lib;
|
||||
rancherDistro = "k3s";
|
||||
};
|
||||
kafka = handleTest ./kafka { };
|
||||
kanboard = runTest ./web-apps/kanboard.nix;
|
||||
kanidm = runTest ./kanidm.nix;
|
||||
@@ -1346,7 +1350,15 @@ in
|
||||
restic-rest-server = runTest ./restic-rest-server.nix;
|
||||
retroarch = runTest ./retroarch.nix;
|
||||
ringboard = runTest ./ringboard.nix;
|
||||
rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; };
|
||||
rke2 = import ./rancher {
|
||||
inherit pkgs;
|
||||
inherit (pkgs) lib;
|
||||
runTest = runTestOn [
|
||||
"aarch64-linux"
|
||||
"x86_64-linux"
|
||||
];
|
||||
rancherDistro = "rke2";
|
||||
};
|
||||
rkvm = handleTest ./rkvm { };
|
||||
rmfakecloud = runTest ./rmfakecloud.nix;
|
||||
robustirc-bridge = runTest ./robustirc-bridge.nix;
|
||||
|
||||
@@ -1,42 +1,40 @@
|
||||
# A test that imports k3s airgapped images and verifies that all expected images are present
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
{
|
||||
name = "${rancherPackage.name}-airgap-images";
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
{
|
||||
name = "${rancherPackage.name}-airgap-images";
|
||||
|
||||
nodes.machine = _: {
|
||||
virtualisation = vmResources;
|
||||
nodes.machine = _: {
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images =
|
||||
coreImages
|
||||
++ {
|
||||
k3s = [ rancherPackage.airgap-images ];
|
||||
rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages
|
||||
}
|
||||
.${rancherDistro};
|
||||
};
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images =
|
||||
coreImages
|
||||
++ {
|
||||
k3s = [ rancherPackage.airgap-images ];
|
||||
rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages
|
||||
}
|
||||
.${rancherDistro};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")
|
||||
'';
|
||||
testScript = ''
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,232 +1,230 @@
|
||||
# Tests whether container images are imported and auto deploying Helm charts,
|
||||
# including the bundled traefik or ingress-nginx, work
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
testImageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
busybox
|
||||
hello
|
||||
];
|
||||
};
|
||||
testImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/test";
|
||||
tag = "local";
|
||||
# Slightly reduces the time needed to import image
|
||||
compressor = "zstd";
|
||||
copyToRoot = testImageEnv;
|
||||
};
|
||||
# pack the test helm chart as a .tgz archive
|
||||
package =
|
||||
pkgs.runCommand "${rancherDistro}-test-chart.tgz"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.kubernetes-helm ];
|
||||
chart = builtins.toJSON {
|
||||
name = "${rancherDistro}-test-chart";
|
||||
version = "0.1.0";
|
||||
};
|
||||
values = builtins.toJSON {
|
||||
restartPolicy = "Never";
|
||||
runCommand = "";
|
||||
image = {
|
||||
repository = "foo";
|
||||
tag = "1.0.0";
|
||||
};
|
||||
};
|
||||
job = builtins.toJSON {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata = {
|
||||
name = "{{ .Release.Name }}";
|
||||
namespace = "{{ .Release.Namespace }}";
|
||||
};
|
||||
spec = {
|
||||
template = {
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "test";
|
||||
image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}";
|
||||
command = [ "sh" ];
|
||||
args = [
|
||||
"-c"
|
||||
"{{ .Values.runCommand }}"
|
||||
];
|
||||
}
|
||||
];
|
||||
restartPolicy = "{{ .Values.restartPolicy }}";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
passAsFile = [
|
||||
"values"
|
||||
"chart"
|
||||
"job"
|
||||
];
|
||||
}
|
||||
''
|
||||
mkdir -p chart/templates
|
||||
cp "$chartPath" chart/Chart.yaml
|
||||
cp "$valuesPath" chart/values.yaml
|
||||
cp "$jobPath" chart/templates/job.json
|
||||
|
||||
helm package chart
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
# The common Helm chart that is used in this test
|
||||
testChart = {
|
||||
inherit package;
|
||||
values = {
|
||||
runCommand = "hello";
|
||||
image = {
|
||||
repository = testImage.imageName;
|
||||
tag = testImage.imageTag;
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-auto-deploy-helm";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
testImageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
busybox
|
||||
hello
|
||||
];
|
||||
};
|
||||
testImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/test";
|
||||
tag = "local";
|
||||
# Slightly reduces the time needed to import image
|
||||
compressor = "zstd";
|
||||
copyToRoot = testImageEnv;
|
||||
};
|
||||
# pack the test helm chart as a .tgz archive
|
||||
package =
|
||||
pkgs.runCommand "${rancherDistro}-test-chart.tgz"
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
yq-go
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable =
|
||||
{
|
||||
k3s = lib.remove "traefik" disabledComponents;
|
||||
rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;
|
||||
}
|
||||
.${rancherDistro};
|
||||
images =
|
||||
coreImages
|
||||
# Provides the k3s Helm controller
|
||||
++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images
|
||||
++ [
|
||||
testImage
|
||||
];
|
||||
autoDeployCharts = {
|
||||
# regular test chart that should get installed
|
||||
hello = testChart;
|
||||
# disabled chart that should not get installed
|
||||
disabled = testChart // {
|
||||
enable = false;
|
||||
};
|
||||
# chart with values set via YAML file
|
||||
values-file = testChart // {
|
||||
# Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
|
||||
# https://github.com/NixOS/nix/issues/12407
|
||||
values =
|
||||
/.
|
||||
+ builtins.unsafeDiscardStringContext (
|
||||
builtins.toFile "${rancherDistro}-test-chart-values.yaml" ''
|
||||
runCommand: "echo 'Hello, file!'"
|
||||
image:
|
||||
repository: test.local/test
|
||||
tag: local
|
||||
''
|
||||
);
|
||||
};
|
||||
# advanced chart that should get installed in the "test" namespace with a custom
|
||||
# timeout and overridden values
|
||||
advanced = testChart // {
|
||||
# create the "test" namespace via extraDeploy for testing
|
||||
extraDeploy = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "test";
|
||||
}
|
||||
];
|
||||
extraFieldDefinitions = {
|
||||
spec = {
|
||||
# overwrite chart values
|
||||
valuesContent = ''
|
||||
runCommand: "echo 'advanced hello'"
|
||||
image:
|
||||
repository: ${testImage.imageName}
|
||||
tag: ${testImage.imageTag}
|
||||
'';
|
||||
# overwrite the chart namespace
|
||||
targetNamespace = "test";
|
||||
# set a custom timeout
|
||||
timeout = "69s";
|
||||
};
|
||||
nativeBuildInputs = [ pkgs.kubernetes-helm ];
|
||||
chart = builtins.toJSON {
|
||||
name = "${rancherDistro}-test-chart";
|
||||
version = "0.1.0";
|
||||
};
|
||||
values = builtins.toJSON {
|
||||
restartPolicy = "Never";
|
||||
runCommand = "";
|
||||
image = {
|
||||
repository = "foo";
|
||||
tag = "1.0.0";
|
||||
};
|
||||
};
|
||||
job = builtins.toJSON {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata = {
|
||||
name = "{{ .Release.Name }}";
|
||||
namespace = "{{ .Release.Namespace }}";
|
||||
};
|
||||
spec = {
|
||||
template = {
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "test";
|
||||
image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}";
|
||||
command = [ "sh" ];
|
||||
args = [
|
||||
"-c"
|
||||
"{{ .Values.runCommand }}"
|
||||
];
|
||||
}
|
||||
];
|
||||
restartPolicy = "{{ .Values.restartPolicy }}";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
passAsFile = [
|
||||
"values"
|
||||
"chart"
|
||||
"job"
|
||||
];
|
||||
}
|
||||
''
|
||||
mkdir -p chart/templates
|
||||
cp "$chartPath" chart/Chart.yaml
|
||||
cp "$valuesPath" chart/values.yaml
|
||||
cp "$jobPath" chart/templates/job.json
|
||||
|
||||
testScript = # python
|
||||
let
|
||||
manifestFormat =
|
||||
helm package chart
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
# The common Helm chart that is used in this test
|
||||
testChart = {
|
||||
inherit package;
|
||||
values = {
|
||||
runCommand = "hello";
|
||||
image = {
|
||||
repository = testImage.imageName;
|
||||
tag = testImage.imageTag;
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-auto-deploy-helm";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
yq-go
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable =
|
||||
{
|
||||
k3s = "yaml";
|
||||
rke2 = "json";
|
||||
k3s = lib.remove "traefik" disabledComponents;
|
||||
rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;
|
||||
}
|
||||
.${rancherDistro};
|
||||
in
|
||||
''
|
||||
import json
|
||||
images =
|
||||
coreImages
|
||||
# Provides the k3s Helm controller
|
||||
++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images
|
||||
++ [
|
||||
testImage
|
||||
];
|
||||
autoDeployCharts = {
|
||||
# regular test chart that should get installed
|
||||
hello = testChart;
|
||||
# disabled chart that should not get installed
|
||||
disabled = testChart // {
|
||||
enable = false;
|
||||
};
|
||||
# chart with values set via YAML file
|
||||
values-file = testChart // {
|
||||
# Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
|
||||
# https://github.com/NixOS/nix/issues/12407
|
||||
values =
|
||||
/.
|
||||
+ builtins.unsafeDiscardStringContext (
|
||||
builtins.toFile "${rancherDistro}-test-chart-values.yaml" ''
|
||||
runCommand: "echo 'Hello, file!'"
|
||||
image:
|
||||
repository: test.local/test
|
||||
tag: local
|
||||
''
|
||||
);
|
||||
};
|
||||
# advanced chart that should get installed in the "test" namespace with a custom
|
||||
# timeout and overridden values
|
||||
advanced = testChart // {
|
||||
# create the "test" namespace via extraDeploy for testing
|
||||
extraDeploy = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "test";
|
||||
}
|
||||
];
|
||||
extraFieldDefinitions = {
|
||||
spec = {
|
||||
# overwrite chart values
|
||||
valuesContent = ''
|
||||
runCommand: "echo 'advanced hello'"
|
||||
image:
|
||||
repository: ${testImage.imageName}
|
||||
tag: ${testImage.imageTag}
|
||||
'';
|
||||
# overwrite the chart namespace
|
||||
targetNamespace = "test";
|
||||
# set a custom timeout
|
||||
timeout = "69s";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# check existence/absence of chart manifest files
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
|
||||
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}")
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}")
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")
|
||||
# check that the timeout is set correctly, select only the first item in advanced.yaml
|
||||
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}"))
|
||||
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
|
||||
# wait for test jobs to complete
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
|
||||
# check output of test jobs
|
||||
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
|
||||
values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
|
||||
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
|
||||
# strip the output to remove trailing whitespaces
|
||||
t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
|
||||
t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
|
||||
t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
|
||||
# wait for bundled ingress deployment
|
||||
${
|
||||
{
|
||||
k3s = ''
|
||||
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
|
||||
'';
|
||||
rke2 = ''
|
||||
machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180)
|
||||
'';
|
||||
}
|
||||
.${rancherDistro}
|
||||
testScript = # python
|
||||
let
|
||||
manifestFormat =
|
||||
{
|
||||
k3s = "yaml";
|
||||
rke2 = "json";
|
||||
}
|
||||
'';
|
||||
.${rancherDistro};
|
||||
in
|
||||
''
|
||||
import json
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# check existence/absence of chart manifest files
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
|
||||
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}")
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}")
|
||||
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")
|
||||
# check that the timeout is set correctly, select only the first item in advanced.yaml
|
||||
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}"))
|
||||
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
|
||||
# wait for test jobs to complete
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
|
||||
# check output of test jobs
|
||||
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
|
||||
values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
|
||||
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
|
||||
# strip the output to remove trailing whitespaces
|
||||
t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
|
||||
t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
|
||||
t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
|
||||
# wait for bundled ingress deployment
|
||||
${
|
||||
{
|
||||
k3s = ''
|
||||
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
|
||||
'';
|
||||
rke2 = ''
|
||||
machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180)
|
||||
'';
|
||||
}
|
||||
.${rancherDistro}
|
||||
}
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,136 +1,134 @@
|
||||
# Tests whether container images are imported and auto deploying manifests work
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
pauseImageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(lib.hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = pauseImageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
helloImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/hello";
|
||||
tag = "local";
|
||||
copyToRoot = pkgs.hello;
|
||||
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
|
||||
};
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
pauseImageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(lib.hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = pauseImageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
helloImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/hello";
|
||||
tag = "local";
|
||||
copyToRoot = pkgs.hello;
|
||||
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
|
||||
};
|
||||
|
||||
manifestFormat =
|
||||
{
|
||||
k3s = "yaml";
|
||||
rke2 = "json";
|
||||
}
|
||||
.${rancherDistro};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-auto-deploy";
|
||||
manifestFormat =
|
||||
{
|
||||
k3s = "yaml";
|
||||
rke2 = "json";
|
||||
}
|
||||
.${rancherDistro};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-auto-deploy";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
cri-tools
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
cri-tools
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
images = coreImages ++ [
|
||||
pauseImage
|
||||
helloImage
|
||||
];
|
||||
manifests = {
|
||||
absent = {
|
||||
enable = false;
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "absent";
|
||||
};
|
||||
images = coreImages ++ [
|
||||
pauseImage
|
||||
helloImage
|
||||
];
|
||||
manifests = {
|
||||
absent = {
|
||||
enable = false;
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "absent";
|
||||
};
|
||||
};
|
||||
|
||||
present = {
|
||||
target = "foo-namespace.${manifestFormat}";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "foo";
|
||||
};
|
||||
present = {
|
||||
target = "foo-namespace.${manifestFormat}";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "foo";
|
||||
};
|
||||
};
|
||||
|
||||
hello.content = {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata.name = "hello";
|
||||
spec = {
|
||||
template.spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "hello";
|
||||
image = "test.local/hello:local";
|
||||
}
|
||||
];
|
||||
restartPolicy = "OnFailure";
|
||||
};
|
||||
hello.content = {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata.name = "hello";
|
||||
spec = {
|
||||
template.spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "hello";
|
||||
image = "test.local/hello:local";
|
||||
}
|
||||
];
|
||||
restartPolicy = "OnFailure";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# check existence of the manifest files
|
||||
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}")
|
||||
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}")
|
||||
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# check existence of the manifest files
|
||||
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}")
|
||||
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}")
|
||||
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
|
||||
|
||||
# check if container images got imported
|
||||
# for some reason, RKE2 also uses /run/k3s
|
||||
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'")
|
||||
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'")
|
||||
# check if container images got imported
|
||||
# for some reason, RKE2 also uses /run/k3s
|
||||
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'")
|
||||
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'")
|
||||
|
||||
# check if resources of manifests got created
|
||||
machine.wait_until_succeeds("kubectl get ns foo")
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
|
||||
machine.fail("kubectl get ns absent")
|
||||
'';
|
||||
# check if resources of manifests got created
|
||||
machine.wait_until_succeeds("kubectl get ns foo")
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
|
||||
machine.fail("kubectl get ns absent")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,61 +1,59 @@
|
||||
# A test that containerdConfigTemplate settings get written to containerd/config.toml
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
nodeName = "test";
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-containerd-config";
|
||||
nodes.machine =
|
||||
{ ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
nodeName = "test";
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-containerd-config";
|
||||
nodes.machine =
|
||||
{ ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
inherit nodeName;
|
||||
containerdConfigTemplate = ''
|
||||
# Base ${rancherDistro} config
|
||||
{{ template "base" . }}
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
inherit nodeName;
|
||||
containerdConfigTemplate = ''
|
||||
# Base ${rancherDistro} config
|
||||
{{ template "base" . }}
|
||||
|
||||
# MAGIC COMMENT
|
||||
'';
|
||||
};
|
||||
# MAGIC COMMENT
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# wait until the node is ready
|
||||
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
|
||||
# test whether the config template file contains the magic comment
|
||||
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl")
|
||||
t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment")
|
||||
# test whether the config file contains the magic comment
|
||||
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml")
|
||||
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
|
||||
'';
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# wait until the node is ready
|
||||
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
|
||||
# test whether the config template file contains the magic comment
|
||||
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl")
|
||||
t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment")
|
||||
# test whether the config file contains the magic comment
|
||||
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml")
|
||||
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,101 +1,116 @@
|
||||
{
|
||||
system ? builtins.currentSystem,
|
||||
pkgs ? import ../../.. { inherit system; },
|
||||
lib ? pkgs.lib,
|
||||
runTest,
|
||||
pkgs,
|
||||
lib,
|
||||
# service/package name to test
|
||||
rancherDistro,
|
||||
...
|
||||
}:
|
||||
let
|
||||
allPackages = lib.filterAttrs (
|
||||
n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success
|
||||
name: package:
|
||||
builtins.match "^${rancherDistro}(_[[:digit:]]+)+$" name != null
|
||||
&& (builtins.tryEval package).success
|
||||
) pkgs;
|
||||
|
||||
mkTestArgs = rancherPackage: {
|
||||
inherit
|
||||
system
|
||||
pkgs
|
||||
rancherDistro
|
||||
rancherPackage
|
||||
;
|
||||
allTests =
|
||||
let
|
||||
mkTestArgs = rancherPackage: {
|
||||
inherit rancherDistro rancherPackage;
|
||||
|
||||
# systemd service name
|
||||
serviceName =
|
||||
{
|
||||
k3s = "k3s";
|
||||
rke2 = "rke2-server";
|
||||
}
|
||||
.${rancherDistro};
|
||||
|
||||
# list passed to services.*.disable,
|
||||
# for slightly reduced resource usage
|
||||
disabledComponents =
|
||||
{
|
||||
k3s = [
|
||||
"coredns"
|
||||
"local-storage"
|
||||
"metrics-server"
|
||||
"servicelb"
|
||||
"traefik"
|
||||
];
|
||||
rke2 = [
|
||||
"rke2-coredns"
|
||||
"rke2-metrics-server"
|
||||
"rke2-ingress-nginx"
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
}
|
||||
.${rancherDistro};
|
||||
|
||||
# images that must be present for all tests
|
||||
coreImages =
|
||||
{
|
||||
k3s = [ ];
|
||||
|
||||
rke2 =
|
||||
# systemd service name
|
||||
serviceName =
|
||||
{
|
||||
aarch64-linux = [
|
||||
rancherPackage.images-core-linux-arm64-tar-zst
|
||||
rancherPackage.images-canal-linux-arm64-tar-zst
|
||||
k3s = "k3s";
|
||||
rke2 = "rke2-server";
|
||||
}
|
||||
.${rancherDistro};
|
||||
|
||||
# list passed to services.*.disable,
|
||||
# for slightly reduced resource usage
|
||||
disabledComponents =
|
||||
{
|
||||
k3s = [
|
||||
"coredns"
|
||||
"local-storage"
|
||||
"metrics-server"
|
||||
"servicelb"
|
||||
"traefik"
|
||||
];
|
||||
x86_64-linux = [
|
||||
rancherPackage.images-core-linux-amd64-tar-zst
|
||||
rancherPackage.images-canal-linux-amd64-tar-zst
|
||||
rke2 = [
|
||||
"rke2-coredns"
|
||||
"rke2-metrics-server"
|
||||
"rke2-ingress-nginx"
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
}
|
||||
.${pkgs.stdenv.hostPlatform.system}
|
||||
or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}");
|
||||
}
|
||||
.${rancherDistro};
|
||||
.${rancherDistro};
|
||||
|
||||
# virtualization.* attrs, since all distros
|
||||
# need more resources than the default
|
||||
vmResources =
|
||||
{
|
||||
k3s = {
|
||||
memorySize = 1536;
|
||||
diskSize = 4096;
|
||||
};
|
||||
rke2 = {
|
||||
cores = 4;
|
||||
memorySize = 4096;
|
||||
diskSize = 8092;
|
||||
};
|
||||
}
|
||||
.${rancherDistro};
|
||||
};
|
||||
# images that must be present for all tests
|
||||
coreImages =
|
||||
{
|
||||
k3s = [ ];
|
||||
|
||||
importTest =
|
||||
path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages;
|
||||
rke2 =
|
||||
{
|
||||
aarch64-linux = [
|
||||
rancherPackage.images-core-linux-arm64-tar-zst
|
||||
rancherPackage.images-canal-linux-arm64-tar-zst
|
||||
];
|
||||
x86_64-linux = [
|
||||
rancherPackage.images-core-linux-amd64-tar-zst
|
||||
rancherPackage.images-canal-linux-amd64-tar-zst
|
||||
];
|
||||
}
|
||||
.${pkgs.stdenv.hostPlatform.system}
|
||||
or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}");
|
||||
}
|
||||
.${rancherDistro};
|
||||
|
||||
# virtualization.* attrs, since all distros
|
||||
# need more resources than the default
|
||||
vmResources =
|
||||
{
|
||||
k3s = {
|
||||
memorySize = 1536;
|
||||
diskSize = 4096;
|
||||
};
|
||||
rke2 = {
|
||||
cores = 4;
|
||||
memorySize = 4096;
|
||||
diskSize = 8092;
|
||||
};
|
||||
}
|
||||
.${rancherDistro};
|
||||
};
|
||||
|
||||
mkTests =
|
||||
path:
|
||||
lib.mapAttrs (
|
||||
name: package:
|
||||
runTest {
|
||||
imports = [ path ];
|
||||
_module.args = mkTestArgs package;
|
||||
}
|
||||
) allPackages;
|
||||
in
|
||||
{
|
||||
airgap-images = mkTests ./airgap-images.nix;
|
||||
auto-deploy = mkTests ./auto-deploy.nix;
|
||||
auto-deploy-charts = mkTests ./auto-deploy-charts.nix;
|
||||
containerd-config = mkTests ./containerd-config.nix;
|
||||
etcd = mkTests ./etcd.nix;
|
||||
kubelet-config = mkTests ./kubelet-config.nix;
|
||||
multi-node = mkTests ./multi-node.nix;
|
||||
single-node = mkTests ./single-node.nix;
|
||||
};
|
||||
in
|
||||
{
|
||||
airgap-images = importTest ./airgap-images.nix { };
|
||||
auto-deploy = importTest ./auto-deploy.nix { };
|
||||
auto-deploy-charts = importTest ./auto-deploy-charts.nix { };
|
||||
containerd-config = importTest ./containerd-config.nix { };
|
||||
etcd = importTest ./etcd.nix { };
|
||||
kubelet-config = importTest ./kubelet-config.nix { };
|
||||
multi-node = importTest ./multi-node.nix { };
|
||||
single-node = importTest ./single-node.nix { };
|
||||
|
||||
allTests
|
||||
// {
|
||||
all = lib.concatMapAttrs (
|
||||
testType: lib.mapAttrs' (package: lib.nameValuePair "${testType}-${package}")
|
||||
) allTests;
|
||||
}
|
||||
|
||||
@@ -1,131 +1,129 @@
|
||||
# Tests K3s with Etcd backend
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
|
||||
{
|
||||
name = "${rancherPackage.name}-etcd";
|
||||
{
|
||||
name = "${rancherPackage.name}-etcd";
|
||||
|
||||
nodes = {
|
||||
nodes = {
|
||||
|
||||
etcd =
|
||||
{ ... }:
|
||||
{
|
||||
services.etcd = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
listenClientUrls = [
|
||||
"http://192.168.1.1:2379"
|
||||
"http://127.0.0.1:2379"
|
||||
];
|
||||
listenPeerUrls = [ "http://192.168.1.1:2380" ];
|
||||
initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
|
||||
initialCluster = [ "etcd=http://192.168.1.1:2380" ];
|
||||
};
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
defaultGateway = "192.168.1.1";
|
||||
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
|
||||
{
|
||||
address = "192.168.1.1";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
server =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
etcd =
|
||||
{ ... }:
|
||||
{
|
||||
services.etcd = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
listenClientUrls = [
|
||||
"http://192.168.1.1:2379"
|
||||
"http://127.0.0.1:2379"
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
nodeIP = "192.168.1.2";
|
||||
extraFlags = [
|
||||
"--datastore-endpoint=\"http://192.168.1.1:2379\""
|
||||
];
|
||||
};
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
];
|
||||
allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
useDHCP = false;
|
||||
defaultGateway = "192.168.1.2";
|
||||
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
|
||||
{
|
||||
address = "192.168.1.2";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
listenPeerUrls = [ "http://192.168.1.1:2380" ];
|
||||
initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
|
||||
initialCluster = [ "etcd=http://192.168.1.1:2380" ];
|
||||
};
|
||||
};
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
defaultGateway = "192.168.1.1";
|
||||
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
|
||||
{
|
||||
address = "192.168.1.1";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
with subtest("should start etcd"):
|
||||
etcd.start()
|
||||
etcd.wait_for_unit("etcd.service")
|
||||
server =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
with subtest("should wait for etcdctl endpoint status to succeed"):
|
||||
etcd.wait_until_succeeds("etcdctl endpoint status")
|
||||
virtualisation = vmResources;
|
||||
|
||||
with subtest("should wait for etcdctl endpoint health to succeed"):
|
||||
etcd.wait_until_succeeds("etcdctl endpoint health")
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
nodeIP = "192.168.1.2";
|
||||
extraFlags = [
|
||||
"--datastore-endpoint=\"http://192.168.1.1:2379\""
|
||||
];
|
||||
};
|
||||
|
||||
with subtest("should start ${rancherDistro}"):
|
||||
server.start()
|
||||
server.wait_for_unit("${serviceName}")
|
||||
networking = {
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
];
|
||||
allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
useDHCP = false;
|
||||
defaultGateway = "192.168.1.2";
|
||||
interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
|
||||
{
|
||||
address = "192.168.1.2";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
with subtest("should test if kubectl works"):
|
||||
server.wait_until_succeeds("kubectl get node")
|
||||
testScript = # python
|
||||
''
|
||||
with subtest("should start etcd"):
|
||||
etcd.start()
|
||||
etcd.wait_for_unit("etcd.service")
|
||||
|
||||
with subtest("should wait for service account to show up; takes a sec"):
|
||||
server.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
with subtest("should wait for etcdctl endpoint status to succeed"):
|
||||
etcd.wait_until_succeeds("etcdctl endpoint status")
|
||||
|
||||
with subtest("should create a sample secret object"):
|
||||
server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
|
||||
with subtest("should wait for etcdctl endpoint health to succeed"):
|
||||
etcd.wait_until_succeeds("etcdctl endpoint health")
|
||||
|
||||
with subtest("should check if secret is correct"):
|
||||
server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
|
||||
with subtest("should start ${rancherDistro}"):
|
||||
server.start()
|
||||
server.wait_for_unit("${serviceName}")
|
||||
|
||||
with subtest("should have a secret in database"):
|
||||
etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
|
||||
with subtest("should test if kubectl works"):
|
||||
server.wait_until_succeeds("kubectl get node")
|
||||
|
||||
with subtest("should delete the secret"):
|
||||
server.succeed("kubectl delete secret nixossecret")
|
||||
with subtest("should wait for service account to show up; takes a sec"):
|
||||
server.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
|
||||
with subtest("should not have a secret in database"):
|
||||
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
|
||||
'';
|
||||
with subtest("should create a sample secret object"):
|
||||
server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
|
||||
|
||||
meta.maintainers =
|
||||
pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
with subtest("should check if secret is correct"):
|
||||
server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
|
||||
|
||||
with subtest("should have a secret in database"):
|
||||
etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
|
||||
|
||||
with subtest("should delete the secret"):
|
||||
server.succeed("kubectl delete secret nixossecret")
|
||||
|
||||
with subtest("should not have a secret in database"):
|
||||
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
|
||||
'';
|
||||
|
||||
meta.maintainers =
|
||||
pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,77 +1,75 @@
|
||||
# A test that sets extra kubelet configuration and enables graceful node shutdown
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
nodeName = "test";
|
||||
shutdownGracePeriod = "1m13s";
|
||||
shutdownGracePeriodCriticalPods = "13s";
|
||||
podsPerCore = 3;
|
||||
memoryThrottlingFactor = 0.69;
|
||||
containerLogMaxSize = "5Mi";
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-kubelet-config";
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
nodeName = "test";
|
||||
shutdownGracePeriod = "1m13s";
|
||||
shutdownGracePeriodCriticalPods = "13s";
|
||||
podsPerCore = 3;
|
||||
memoryThrottlingFactor = 0.69;
|
||||
containerLogMaxSize = "5Mi";
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-kubelet-config";
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
inherit nodeName;
|
||||
gracefulNodeShutdown = {
|
||||
enable = true;
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages;
|
||||
inherit nodeName;
|
||||
gracefulNodeShutdown = {
|
||||
enable = true;
|
||||
inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
|
||||
};
|
||||
extraKubeletConfig = {
|
||||
inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
|
||||
};
|
||||
inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
|
||||
};
|
||||
extraKubeletConfig = {
|
||||
inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
import json
|
||||
testScript = # python
|
||||
''
|
||||
import json
|
||||
|
||||
start_all()
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# wait until the node is ready
|
||||
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
|
||||
# test whether the kubelet registered an inhibitor lock
|
||||
machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"")
|
||||
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
|
||||
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
|
||||
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
|
||||
# get the kubeletconfig
|
||||
kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
|
||||
start_all()
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
# wait until the node is ready
|
||||
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
|
||||
# test whether the kubelet registered an inhibitor lock
|
||||
machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"")
|
||||
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
|
||||
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
|
||||
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
|
||||
# get the kubeletconfig
|
||||
kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
|
||||
|
||||
with subtest("Kubelet config values are set correctly"):
|
||||
t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}")
|
||||
t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}")
|
||||
t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore})
|
||||
t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor})
|
||||
t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
|
||||
'';
|
||||
with subtest("Kubelet config values are set correctly"):
|
||||
t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}")
|
||||
t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}")
|
||||
t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore})
|
||||
t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor})
|
||||
t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,252 +1,250 @@
|
||||
# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
imageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
bashInteractive
|
||||
coreutils
|
||||
socat
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = imageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
# A daemonset that responds 'server' on port 8000
|
||||
networkTestDaemonset = pkgs.writeText "test.yml" ''
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
imageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
bashInteractive
|
||||
coreutils
|
||||
socat
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = imageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
# A daemonset that responds 'server' on port 8000
|
||||
networkTestDaemonset = pkgs.writeText "test.yml" ''
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: test
|
||||
labels:
|
||||
name: test
|
||||
labels:
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
name: test
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: test
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: test
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: test.local/pause:local
|
||||
imagePullPolicy: Never
|
||||
resources:
|
||||
limits:
|
||||
memory: 20Mi
|
||||
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
|
||||
'';
|
||||
tokenFile = pkgs.writeText "token" "p@s$w0rd";
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: test.local/pause:local
|
||||
imagePullPolicy: Never
|
||||
resources:
|
||||
limits:
|
||||
memory: 20Mi
|
||||
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
|
||||
'';
|
||||
tokenFile = pkgs.writeText "token" "p@s$w0rd";
|
||||
|
||||
supervisorPort =
|
||||
supervisorPort =
|
||||
{
|
||||
k3s = "6443";
|
||||
rke2 = "9345";
|
||||
}
|
||||
.${rancherDistro};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-multi-node";
|
||||
|
||||
nodes = {
|
||||
server =
|
||||
{
|
||||
k3s = "6443";
|
||||
rke2 = "9345";
|
||||
}
|
||||
.${rancherDistro};
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-multi-node";
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
gzip
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
nodes = {
|
||||
server =
|
||||
{
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
gzip
|
||||
jq
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
virtualisation = vmResources;
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = lib.mkMerge [
|
||||
{
|
||||
inherit tokenFile;
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
nodeIP = config.networking.primaryIPAddress;
|
||||
disable = disabledComponents;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
}
|
||||
{
|
||||
k3s = {
|
||||
clusterInit = true;
|
||||
extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition
|
||||
};
|
||||
|
||||
# The interface selection logic of flannel & canal would normally use eth0, as
|
||||
# the nixos testing driver sets a default route via dev eth0. However, in test
|
||||
# setups we have to use eth1 for inter-node communication.
|
||||
# For K3s this can be handled via --flannel-iface, but RKE2's canal has to be
|
||||
# configured with this manifest.
|
||||
rke2.manifests.canal-config.content = {
|
||||
apiVersion = "helm.cattle.io/v1";
|
||||
kind = "HelmChartConfig";
|
||||
metadata = {
|
||||
name = "rke2-canal";
|
||||
namespace = "kube-system";
|
||||
};
|
||||
# spec.valuesContent needs to a string, either json or yaml
|
||||
spec.valuesContent = builtins.toJSON {
|
||||
flannel.iface = "eth1";
|
||||
};
|
||||
};
|
||||
}
|
||||
.${rancherDistro}
|
||||
];
|
||||
|
||||
networking.firewall.enable = false;
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
]
|
||||
++ lib.optionals (rancherDistro == "rke2") [
|
||||
9099
|
||||
9345
|
||||
];
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
|
||||
server2 =
|
||||
{
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
services.${rancherDistro} = lib.mkMerge [
|
||||
{
|
||||
inherit tokenFile;
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}";
|
||||
nodeIP = config.networking.primaryIPAddress;
|
||||
disable = disabledComponents;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
]
|
||||
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
|
||||
};
|
||||
];
|
||||
}
|
||||
{
|
||||
k3s = {
|
||||
clusterInit = true;
|
||||
extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition
|
||||
};
|
||||
|
||||
networking.firewall.enable = false;
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
# The interface selection logic of flannel & canal would normally use eth0, as
|
||||
# the nixos testing driver sets a default route via dev eth0. However, in test
|
||||
# setups we have to use eth1 for inter-node communication.
|
||||
# For K3s this can be handled via --flannel-iface, but RKE2's canal has to be
|
||||
# configured with this manifest.
|
||||
rke2.manifests.canal-config.content = {
|
||||
apiVersion = "helm.cattle.io/v1";
|
||||
kind = "HelmChartConfig";
|
||||
metadata = {
|
||||
name = "rke2-canal";
|
||||
namespace = "kube-system";
|
||||
};
|
||||
# spec.valuesContent needs to a string, either json or yaml
|
||||
spec.valuesContent = builtins.toJSON {
|
||||
flannel.iface = "eth1";
|
||||
};
|
||||
};
|
||||
}
|
||||
.${rancherDistro}
|
||||
];
|
||||
|
||||
networking.firewall.enable = false;
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
]
|
||||
++ lib.optionals (rancherDistro == "rke2") [
|
||||
9099
|
||||
9345
|
||||
];
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
|
||||
server2 =
|
||||
{
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
inherit tokenFile;
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}";
|
||||
nodeIP = config.networking.primaryIPAddress;
|
||||
disable = disabledComponents;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
]
|
||||
++ lib.optionals (rancherDistro == "rke2") [
|
||||
9099
|
||||
9345
|
||||
];
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
|
||||
};
|
||||
|
||||
agent =
|
||||
{
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
virtualisation = vmResources;
|
||||
networking.firewall.enable = false;
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379
|
||||
2380
|
||||
6443
|
||||
]
|
||||
++ lib.optionals (rancherDistro == "rke2") [
|
||||
9099
|
||||
9345
|
||||
];
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
|
||||
services.${rancherDistro} = {
|
||||
inherit tokenFile;
|
||||
enable = true;
|
||||
role = "agent";
|
||||
package = rancherPackage;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}";
|
||||
nodeIP = config.networking.primaryIPAddress;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
]
|
||||
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
|
||||
};
|
||||
agent =
|
||||
{
|
||||
nodes,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
{
|
||||
virtualisation = vmResources;
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099;
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
services.${rancherDistro} = {
|
||||
inherit tokenFile;
|
||||
enable = true;
|
||||
role = "agent";
|
||||
package = rancherPackage;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}";
|
||||
nodeIP = config.networking.primaryIPAddress;
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
]
|
||||
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099;
|
||||
networking.firewall.allowedUDPPorts = [ 8472 ];
|
||||
};
|
||||
};
|
||||
|
||||
servers = [server, server2]
|
||||
for m in servers:
|
||||
m.wait_for_unit("${serviceName}")
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
|
||||
# wait for the agent to show up
|
||||
server.wait_until_succeeds("kubectl get node agent")
|
||||
servers = [server, server2]
|
||||
for m in servers:
|
||||
m.wait_for_unit("${serviceName}")
|
||||
|
||||
${lib.optionalString (rancherDistro == "k3s") ''
|
||||
for m in machines:
|
||||
m.succeed("k3s check-config")
|
||||
''}
|
||||
# wait for the agent to show up
|
||||
server.wait_until_succeeds("kubectl get node agent")
|
||||
|
||||
server.succeed("kubectl cluster-info")
|
||||
# Also wait for our service account to show up; it takes a sec
|
||||
server.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
${lib.optionalString (rancherDistro == "k3s") ''
|
||||
for m in machines:
|
||||
m.succeed("k3s check-config")
|
||||
''}
|
||||
|
||||
# Now create a pod on each node via a daemonset and verify they can talk to each other.
|
||||
server.succeed("kubectl apply -f ${networkTestDaemonset}")
|
||||
server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
|
||||
server.succeed("kubectl cluster-info")
|
||||
# Also wait for our service account to show up; it takes a sec
|
||||
server.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
|
||||
# Get pod IPs
|
||||
pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
|
||||
pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
|
||||
# Now create a pod on each node via a daemonset and verify they can talk to each other.
|
||||
server.succeed("kubectl apply -f ${networkTestDaemonset}")
|
||||
server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
|
||||
|
||||
# Verify each server can ping each pod ip
|
||||
for pod_ip in pod_ips:
|
||||
server.succeed(f"ping -c 1 {pod_ip}")
|
||||
server2.succeed(f"ping -c 1 {pod_ip}")
|
||||
agent.succeed(f"ping -c 1 {pod_ip}")
|
||||
# Verify the pods can talk to each other
|
||||
for pod in pods:
|
||||
resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
|
||||
t.assertEqual(resp.strip(), "server")
|
||||
'';
|
||||
# Get pod IPs
|
||||
pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
|
||||
pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
# Verify each server can ping each pod ip
|
||||
for pod_ip in pod_ips:
|
||||
server.succeed(f"ping -c 1 {pod_ip}")
|
||||
server2.succeed(f"ping -c 1 {pod_ip}")
|
||||
agent.succeed(f"ping -c 1 {pod_ip}")
|
||||
# Verify the pods can talk to each other
|
||||
for pod in pods:
|
||||
resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
|
||||
t.assertEqual(resp.strip(), "server")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -1,116 +1,114 @@
|
||||
# A test that runs a single node rancher cluster and verifies a pod can run
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
imageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(lib.hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
contents = imageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
testPodYaml = pkgs.writeText "test.yaml" ''
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: test.local/pause:local
|
||||
imagePullPolicy: Never
|
||||
command: ["sh", "-c", "sleep inf"]
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-single-node";
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
rancherDistro,
|
||||
rancherPackage,
|
||||
serviceName,
|
||||
disabledComponents,
|
||||
coreImages,
|
||||
vmResources,
|
||||
...
|
||||
}:
|
||||
let
|
||||
imageEnv = pkgs.buildEnv {
|
||||
name = "${rancherDistro}-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(lib.hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
contents = imageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
testPodYaml = pkgs.writeText "test.yaml" ''
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: test
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: test.local/pause:local
|
||||
imagePullPolicy: Never
|
||||
command: ["sh", "-c", "sleep inf"]
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "${rancherPackage.name}-single-node";
|
||||
|
||||
nodes.machine =
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
gzip
|
||||
nodes.machine =
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [
|
||||
kubectl
|
||||
gzip
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
|
||||
|
||||
virtualisation = vmResources;
|
||||
|
||||
services.${rancherDistro} = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
package = rancherPackage;
|
||||
disable = disabledComponents;
|
||||
images = coreImages ++ [ pauseImage ];
|
||||
extraFlags = [
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
};
|
||||
|
||||
users.users = {
|
||||
noprivs = {
|
||||
isNormalUser = true;
|
||||
description = "Can't access ${rancherDistro} by default";
|
||||
password = "*";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
users.users = {
|
||||
noprivs = {
|
||||
isNormalUser = true;
|
||||
description = "Can't access ${rancherDistro} by default";
|
||||
password = "*";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
machine.succeed("kubectl cluster-info")
|
||||
machine.fail("sudo -u noprivs kubectl cluster-info")
|
||||
${lib.optionalString (rancherDistro == "k3s") ''
|
||||
machine.succeed("k3s check-config")
|
||||
''}
|
||||
testScript = # python
|
||||
''
|
||||
start_all()
|
||||
|
||||
# Also wait for our service account to show up; it takes a sec
|
||||
machine.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
machine.succeed("kubectl apply -f ${testPodYaml}")
|
||||
machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s")
|
||||
machine.succeed("kubectl delete -f ${testPodYaml}")
|
||||
machine.wait_for_unit("${serviceName}")
|
||||
machine.succeed("kubectl cluster-info")
|
||||
machine.fail("sudo -u noprivs kubectl cluster-info")
|
||||
${lib.optionalString (rancherDistro == "k3s") ''
|
||||
machine.succeed("k3s check-config")
|
||||
''}
|
||||
|
||||
# regression test for #176445
|
||||
machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'")
|
||||
# Also wait for our service account to show up; it takes a sec
|
||||
machine.wait_until_succeeds("kubectl get serviceaccount default")
|
||||
machine.succeed("kubectl apply -f ${testPodYaml}")
|
||||
machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s")
|
||||
machine.succeed("kubectl delete -f ${testPodYaml}")
|
||||
|
||||
with subtest("Run ${rancherDistro}-killall"):
|
||||
# Call the killall script with a clean path to assert that
|
||||
# all required commands are wrapped
|
||||
output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr")
|
||||
t.assertNotIn("command not found", output, "killall script contains unknown command")
|
||||
# regression test for #176445
|
||||
machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'")
|
||||
|
||||
# Check that killall cleaned up properly
|
||||
machine.fail("systemctl is-active ${serviceName}.service")
|
||||
machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5)
|
||||
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
|
||||
machine.fail("ip netns show | grep cni-")
|
||||
'';
|
||||
with subtest("Run ${rancherDistro}-killall"):
|
||||
# Call the killall script with a clean path to assert that
|
||||
# all required commands are wrapped
|
||||
output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr")
|
||||
t.assertNotIn("command not found", output, "killall script contains unknown command")
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
)
|
||||
# Check that killall cleaned up properly
|
||||
machine.fail("systemctl is-active ${serviceName}.service")
|
||||
machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5)
|
||||
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
|
||||
machine.fail("ip netns show | grep cni-")
|
||||
'';
|
||||
|
||||
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
|
||||
}
|
||||
|
||||
@@ -470,14 +470,11 @@ buildGoModule (finalAttrs: {
|
||||
;
|
||||
tests =
|
||||
let
|
||||
mkTests =
|
||||
version:
|
||||
let
|
||||
k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version);
|
||||
in
|
||||
lib.mapAttrs (name: value: nixosTests.k3s.${name}.${k3s_version}) nixosTests.k3s;
|
||||
versionedPackage = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor k3sVersion);
|
||||
in
|
||||
mkTests k3sVersion;
|
||||
lib.mapAttrs (name: _: nixosTests.k3s.${name}.${versionedPackage}) (
|
||||
lib.filterAttrs (n: _: n != "all") nixosTests.k3s
|
||||
);
|
||||
imagesList = throw "k3s.imagesList was removed";
|
||||
airgapImages = throw "k3s.airgapImages was renamed to k3s.airgap-images";
|
||||
airgapImagesAmd64 = throw "k3s.airgapImagesAmd64 was renamed to k3s.airgap-images-amd64-tar-zst";
|
||||
|
||||
@@ -22,6 +22,7 @@ lib:
|
||||
makeWrapper,
|
||||
fetchzip,
|
||||
fetchurl,
|
||||
versionCheckHook,
|
||||
|
||||
# Runtime dependencies
|
||||
procps,
|
||||
@@ -42,7 +43,6 @@ lib:
|
||||
|
||||
# Testing dependencies
|
||||
nixosTests,
|
||||
testers,
|
||||
}:
|
||||
buildGoModule (finalAttrs: {
|
||||
pname = "rke2";
|
||||
@@ -129,25 +129,19 @@ buildGoModule (finalAttrs: {
|
||||
go tool nm $out/bin/.rke2-wrapped | grep '_Cfunc__goboringcrypto_' > /dev/null
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
nativeInstallCheckInputs = [ versionCheckHook ];
|
||||
versionCheckProgramArg = "--version";
|
||||
|
||||
passthru = {
|
||||
inherit updateScript;
|
||||
tests =
|
||||
let
|
||||
moduleTests =
|
||||
let
|
||||
package_version =
|
||||
"rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version);
|
||||
in
|
||||
lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2;
|
||||
versionedPackage =
|
||||
"rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version);
|
||||
in
|
||||
{
|
||||
version = testers.testVersion {
|
||||
package = finalAttrs.finalPackage;
|
||||
version = "v${finalAttrs.version}";
|
||||
};
|
||||
}
|
||||
// moduleTests;
|
||||
lib.mapAttrs (name: _: nixosTests.rke2.${name}.${versionedPackage}) (
|
||||
lib.filterAttrs (n: _: n != "all") nixosTests.rke2
|
||||
);
|
||||
}
|
||||
// (lib.mapAttrs (_: value: fetchurl value) imagesVersions);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user