From 559e22bb5936e9d3f4fb86efecd2f085b3801fc9 Mon Sep 17 00:00:00 2001 From: Jonathan Zielinski Date: Thu, 16 Nov 2023 13:24:53 +0100 Subject: [PATCH 1/2] opensearch-dashboards: init at 3.8.0 Co-authored-by: Dom Rodriguez --- .../op/opensearch-dashboards/package.nix | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 pkgs/by-name/op/opensearch-dashboards/package.nix diff --git a/pkgs/by-name/op/opensearch-dashboards/package.nix b/pkgs/by-name/op/opensearch-dashboards/package.nix new file mode 100644 index 000000000000..f26797ee37a0 --- /dev/null +++ b/pkgs/by-name/op/opensearch-dashboards/package.nix @@ -0,0 +1,97 @@ +{ + lib, + stdenv, + fetchurl, + + # nativeBuildInputs + makeWrapper, + + # Runtime deps + nodejs, + coreutils, + which, + + # updateScript + writeShellScript, + curl, + common-updater-scripts, + jq, + + # Optional override for disabling security aspects + disableSecurity ? false, +}: +stdenv.mkDerivation (finalAttrs: { + pname = "opensearch-dashboards"; + version = "3.8.0"; + + src = + finalAttrs.passthru.sources.${stdenv.hostPlatform.system} + or (throw "Unsupported system: ${stdenv.hostPlatform.system}"); + + nativeBuildInputs = [ makeWrapper ]; + + installPhase = '' + mkdir -p $out/libexec/opensearch-dashboards $out/bin + mv * $out/libexec/opensearch-dashboards/ + rm -r $out/libexec/opensearch-dashboards/node + makeWrapper $out/libexec/opensearch-dashboards/bin/opensearch-dashboards $out/bin/opensearch-dashboards \ + --prefix PATH : "${ + lib.makeBinPath [ + nodejs + coreutils + which + ] + }" + sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/opensearch-dashboards/bin/opensearch-dashboards + + ${lib.optionalString disableSecurity "rm -r $out/libexec/opensearch-dashboards/plugins/securityDashboards"} + ''; + + passthru = { + sources = { + "aarch64-linux" = fetchurl { + url = "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/${finalAttrs.version}/opensearch-dashboards-${finalAttrs.version}-linux-arm64.tar.gz"; + hash = "sha256-J6n9u0LmjxCHGELZWnRreHmpre0YYKNsIu+yWOpLmFs="; + }; + "x86_64-linux" = fetchurl { + url = "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/${finalAttrs.version}/opensearch-dashboards-${finalAttrs.version}-linux-x64.tar.gz"; + hash = "sha256-cN9Zd4e1qLt4o7by8i4Xv5VoNTzQ7uh6OPND+s/kqyw="; + }; + }; + updateScript = writeShellScript "update-opensearch-dashboards" '' + set -o errexit + export PATH="${ + lib.makeBinPath [ + curl + jq + common-updater-scripts + ] + }" + NEW_VERSION="$(curl -s "https://api.github.com/repos/opensearch-project/opensearch-dashboards/tags" | jq -r '.[0].name')" + if [[ "${finalAttrs.version}" = "$NEW_VERSION" ]]; then + echo "No update available." + exit 0 + fi + for platform in ${lib.escapeShellArgs finalAttrs.meta.platforms}; do + update-source-version "opensearch-dashboards" "$NEW_VERSION" --ignore-same-version --source-key="sources.$platform" + done + ''; + }; + + __structuredAttrs = true; + strictDeps = true; + + meta = { + changelog = "https://github.com/opensearch-project/OpenSearch-Dashboards/blob/${finalAttrs.version}/release-notes/opensearch-dashboards.release-notes-${finalAttrs.version}.md"; + description = "Visualize logs and time-stamped data"; + downloadPage = "https://github.com/opensearch-project/OpenSearch-Dashboards"; + homepage = "https://opensearch.org/docs/latest/dashboards/index/"; + license = lib.licenses.asl20; + mainProgram = "opensearch-dashboards"; + maintainers = with lib.maintainers; [ + makefu + shymega + ]; + platforms = lib.attrNames finalAttrs.passthru.sources; + }; +}) From 805b3b4e03ddc006906d4bf160f0d54ba4d046a4 Mon Sep 17 00:00:00 2001 From: Jonathan Zielinski Date: Fri, 17 Nov 2023 09:12:50 +0100 Subject: [PATCH 2/2] nixos/opensearch-dashboards: init module Additional changes: - nixos/tests: init opensearch-dashboards test - nixos/doc: Add release note for OpenSearch Dashboards module Co-authored-by: makefu Co-authored-by: Dom Rodriguez --- .../manual/release-notes/rl-2611.section.md | 2 + nixos/modules/module-list.nix | 1 + .../services/search/opensearch-dashboards.nix | 108 ++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/opensearch-dashboards.nix | 117 ++++++++++++++++++ 5 files changed, 229 insertions(+) create mode 100644 nixos/modules/services/search/opensearch-dashboards.nix create mode 100644 nixos/tests/opensearch-dashboards.nix diff --git a/nixos/doc/manual/release-notes/rl-2611.section.md b/nixos/doc/manual/release-notes/rl-2611.section.md index 0d70a142390c..c86b1d24e337 100644 --- a/nixos/doc/manual/release-notes/rl-2611.section.md +++ b/nixos/doc/manual/release-notes/rl-2611.section.md @@ -88,6 +88,8 @@ - [Kener](https://kener.ing), a modern, open-source status page application built with Node.js. Available as [services.kener](#opt-services.kener.enable). +- [OpenSearch Dashboards](https://opensearch.org/platform/opensearch-dashboards/), a powerful, open-source visualization tool that you can use to explore, analyze, and understand your data in real time. Available as [services.opensearch-dashboards](#opt-services.opensearch-dashboards.enable). + - [FlapAlerted](https://github.com/Kioubit/FlapAlerted), detects BGP flapping events and provides statistics based on BGP update messages. Available as [services.flap-alerted](#opt-services.flap-alerted.enable). - [gocron](https://github.com/flohoss/gocron), a task scheduler with web interface. Available as [services.gocron](#opt-services.gocron.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 744cdbf19ad2..7fb0c1e79616 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1545,6 +1545,7 @@ ./services/search/manticore.nix ./services/search/meilisearch.nix ./services/search/nominatim.nix + ./services/search/opensearch-dashboards.nix ./services/search/opensearch.nix ./services/search/qdrant.nix ./services/search/quickwit.nix diff --git a/nixos/modules/services/search/opensearch-dashboards.nix b/nixos/modules/services/search/opensearch-dashboards.nix new file mode 100644 index 000000000000..b5d207399e71 --- /dev/null +++ b/nixos/modules/services/search/opensearch-dashboards.nix @@ -0,0 +1,108 @@ +{ + config, + lib, + options, + pkgs, + utils, + ... +}: +let + inherit (lib) + mkEnableOption + mkIf + mkOption + types + ; + + cfg = config.services.opensearch-dashboards; + yaml = pkgs.formats.yaml { }; + stateDir = "/var/lib/opensearch-dashboards"; +in +{ + meta.maintainers = with lib.maintainers; [ + makefu + shymega + ]; + + options.services.opensearch-dashboards = { + enable = lib.mkEnableOption "OpenSearch Dashboards"; + + package = lib.mkPackageOption pkgs "OpenSearch Dashboards" { + default = [ "opensearch-dashboards" ]; + }; + + user = mkOption { + type = types.str; + default = "opensearch-dashboards"; + description = '' + The user to run OpenSearch Dashboards under. + ''; + }; + + settings = mkOption { + type = types.submodule { + freeformType = yaml.type; + }; + default = { }; + example = lib.literalExpression '' + { + opensearch = { + username = "foo"; + password._secret = "/run/keys/opensearch_password"; + }; + + server = { + name = "your-hostname"; + port = 5601; + maxPayloadbytes = 104857; + }; + }; + ''; + description = '' + OpenSearch Dashboards settings. + + Available settings can be found by looking at the + [opensearch_dashboards.yml](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml) + from the Opensearch Dashboards repo. + + {file}`config/nixos_site_settings.json` file, + attribute set containing the attribute + `_secret` - a string pointing to a file + containing the value the option should be set to. See the + example to get a better picture of this: in the resulting + {file}`opensearch_dashboards.yml` file, + the `opensearch.password` key will + be set to the contents of the + {file}`/run/keys/opensearch_password` + file. + ''; + }; + }; + + config = mkIf (cfg.enable) { + systemd.services.opensearch-dashboards = { + description = "OpenSearch Dashboards Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = { + BABEL_CACHE_PATH = "${stateDir}/.babelcache.json"; + }; + serviceConfig = + let + configPath = stateDir + "/opensearch_dashboards.yml"; + in + { + ExecStartPre = pkgs.writeShellScript "dashboards-exec-pre" '' + set -euo pipefail + umask 077 + ${utils.genJqSecretsReplacementSnippet cfg.settings configPath} + chown ${cfg.user} ${configPath} + ''; + ExecStart = "${cfg.package}/bin/opensearch-dashboards --config ${configPath} --path.data ${stateDir}"; + StateDirectory = "opensearch-dashboards"; + User = cfg.user; + DynamicUser = true; + }; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4d665eb6d08b..386dc9914b98 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1399,6 +1399,7 @@ in openldap = runTest ./openldap.nix; openresty-lua = runTest ./openresty-lua.nix; opensearch = discoverTests (import ./opensearch.nix); + opensearch-dashboards = handleTest ./opensearch-dashboards.nix { }; opensmtpd = handleTest ./opensmtpd.nix { }; opensmtpd-rspamd = handleTest ./opensmtpd-rspamd.nix { }; opensnitch = runTest ./opensnitch.nix; diff --git a/nixos/tests/opensearch-dashboards.nix b/nixos/tests/opensearch-dashboards.nix new file mode 100644 index 000000000000..1f500298e91f --- /dev/null +++ b/nixos/tests/opensearch-dashboards.nix @@ -0,0 +1,117 @@ +import ./make-test-python.nix ( + { pkgs, lib, ... }: + { + name = "opensearch-dashboards"; + meta.maintainers = [ ]; + + nodes.machine = { + virtualisation.memorySize = 4096; + + services.opensearch = { + enable = true; + # security is disabled by default + }; + + services.opensearch-dashboards = { + enable = true; + package = pkgs.opensearch-dashboards.override { + disableSecurity = true; + }; + settings = { + server.host = "0.0.0.0"; + server.port = 5601; + opensearch.hosts = [ "http://localhost:9200" ]; + }; + }; + }; + + testScript = '' + import json + import shlex + + machine.start() + + with subtest("opensearch starts and is reachable"): + machine.wait_for_unit("opensearch.service") + machine.wait_for_open_port(9200) + machine.succeed("curl --fail http://localhost:9200/") + + with subtest("opensearch-dashboards starts and is reachable"): + machine.wait_for_unit("opensearch-dashboards.service") + machine.wait_for_open_port(5601) + # Dashboards takes a while to fully initialize + machine.wait_until_succeeds( + "curl --fail -s http://localhost:5601/api/status" + " | grep -q '\"state\":\"green\"'", + timeout=120, + ) + + with subtest("inject logs into opensearch"): + # Create an index with timestamped log entries + for i in range(5): + doc = json.dumps({ + "@timestamp": f"2026-04-14T10:00:0{i}Z", + "message": f"test log entry {i}", + "level": "info", + "service": "vmtest", + }) + machine.succeed( + "curl --fail -s -X POST http://localhost:9200/test-logs/_doc" + f" -H 'Content-Type: application/json' -d {shlex.quote(doc)}" + ) + + # Refresh the index so documents are searchable + machine.succeed( + "curl --fail -s -X POST http://localhost:9200/test-logs/_refresh" + ) + + # Verify documents are in opensearch + result = machine.succeed( + "curl --fail -s http://localhost:9200/test-logs/_count" + ) + count = json.loads(result)["count"] + assert count == 5, f"Expected 5 documents, got {count}" + + with subtest("create index pattern in dashboards"): + payload = json.dumps({ + "attributes": { + "title": "test-logs*", + "timeFieldName": "@timestamp", + } + }) + machine.succeed( + "curl --fail -s -X POST" + " http://localhost:5601/api/saved_objects/index-pattern/test-logs" + " -H 'osd-xsrf: true'" + f" -H 'Content-Type: application/json' -d {shlex.quote(payload)}" + ) + + with subtest("query logs through dashboards discover search"): + # Use the internal search API that the Discover view uses + query = json.dumps({ + "params": { + "index": "test-logs*", + "body": { + "query": {"match_all": {}}, + "size": 10, + }, + } + }) + result = machine.succeed( + "curl --fail -s -X POST" + " http://localhost:5601/internal/search/opensearch" + " -H 'osd-xsrf: true'" + f" -H 'Content-Type: application/json' -d {shlex.quote(query)}" + ) + data = json.loads(result) + hits = data["rawResponse"]["hits"]["hits"] + assert len(hits) == 5, f"Expected 5 hits from dashboards, got {len(hits)}" + + # Verify the log messages are present + messages = sorted([h["_source"]["message"] for h in hits]) + for i in range(5): + expected = f"test log entry {i}" + assert expected in messages, f"Missing log entry: {expected}" + ''; + } +)