apacheKafka: Add update script (#499399)

pkgs/servers/apache-kafka/3_9.nix

@@ -0,0 +1,5 @@
+import ./generic.nix {
+  kafkaVersion = "3.9.1";
+  scalaVersion = "2.13";
+  hash = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM=";
+}

pkgs/servers/apache-kafka/4_0.nix

@@ -0,0 +1,5 @@
+import ./generic.nix {
+  kafkaVersion = "4.0.1";
+  scalaVersion = "2.13";
+  hash = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ=";
+}

pkgs/servers/apache-kafka/4_1.nix

@@ -0,0 +1,5 @@
+import ./generic.nix {
+  kafkaVersion = "4.1.1";
+  scalaVersion = "2.13";
+  hash = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4=";
+}

pkgs/servers/apache-kafka/default.nix

@@ -1,102 +1,6 @@
+{ callPackage }:
 {
-  lib,
-  stdenv,
-  fetchurl,
-  jdk17_headless,
-  makeWrapper,
-  bash,
-  coreutils,
-  gnugrep,
-  gnused,
-  ps,
-  nixosTests,
-}:
-
-let
-  versionMap = {
-    "4_1" = {
-      kafkaVersion = "4.1.1";
-      scalaVersion = "2.13";
-      sha256 = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4=";
-      jre = jdk17_headless;
-      nixosTest = nixosTests.kafka.base.kafka_4_1;
-    };
-    "4_0" = {
-      kafkaVersion = "4.0.1";
-      scalaVersion = "2.13";
-      sha256 = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ=";
-      jre = jdk17_headless;
-      nixosTest = nixosTests.kafka.base.kafka_4_0;
-    };
-    "3_9" = {
-      kafkaVersion = "3.9.1";
-      scalaVersion = "2.13";
-      sha256 = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM=";
-      jre = jdk17_headless;
-      nixosTest = nixosTests.kafka.base.kafka_3_9;
-    };
-  };
-
-  build =
-    versionInfo:
-    stdenv.mkDerivation rec {
-      version = "${versionInfo.scalaVersion}-${versionInfo.kafkaVersion}";
-      pname = "apache-kafka";
-
-      src = fetchurl {
-        url = "mirror://apache/kafka/${versionInfo.kafkaVersion}/kafka_${version}.tgz";
-        inherit (versionInfo) sha256;
-      };
-
-      nativeBuildInputs = [ makeWrapper ];
-      buildInputs = [
-        versionInfo.jre
-        bash
-        gnugrep
-        gnused
-        coreutils
-        ps
-      ];
-
-      installPhase = ''
-        mkdir -p $out
-        cp -R config libs $out
-
-        mkdir -p $out/bin
-        cp bin/kafka* $out/bin
-        cp bin/connect* $out/bin
-
-        # allow us the specify logging directory using env
-        substituteInPlace $out/bin/kafka-run-class.sh \
-          --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"'
-
-        substituteInPlace $out/bin/kafka-server-stop.sh \
-          --replace 'ps' '${ps}/bin/ps'
-
-        for p in $out/bin\/*.sh; do
-          wrapProgram $p \
-            --set JAVA_HOME "${versionInfo.jre}" \
-            --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \
-            --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin"
-        done
-        chmod +x $out/bin\/*
-      '';
-
-      passthru = {
-        inherit (versionInfo) jre; # Used by the NixOS module to select the supported JRE
-        tests.nixos = versionInfo.nixosTest;
-      };
-
-      meta = {
-        homepage = "https://kafka.apache.org";
-        description = "High-throughput distributed messaging system";
-        license = lib.licenses.asl20;
-        sourceProvenance = [ lib.sourceTypes.binaryBytecode ];
-        maintainers = [ lib.maintainers.ragge ];
-        platforms = lib.platforms.unix;
-      };
-    };
-in
-lib.mapAttrs' (
-  majorVersion: versionInfo: lib.nameValuePair "apacheKafka_${majorVersion}" (build versionInfo)
-) versionMap
+  apacheKafka_4_1 = callPackage ./4_1.nix { };
+  apacheKafka_4_0 = callPackage ./4_0.nix { };
+  apacheKafka_3_9 = callPackage ./3_9.nix { };
+}

pkgs/servers/apache-kafka/generic.nix

@@ -0,0 +1,86 @@
+{
+  kafkaVersion,
+  scalaVersion,
+  hash,
+}:
+
+{
+  lib,
+  stdenv,
+  fetchurl,
+  jdk17_headless,
+  makeWrapper,
+  bash,
+  coreutils,
+  gnugrep,
+  gnused,
+  ps,
+  nixosTests,
+}:
+
+let
+  jre = jdk17_headless;
+  series = lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kafkaVersion);
+  seriesFile = ./. + "/${series}.nix";
+in
+stdenv.mkDerivation rec {
+  version = "${scalaVersion}-${kafkaVersion}";
+  pname = "apache-kafka";
+
+  src = fetchurl {
+    url = "mirror://apache/kafka/${kafkaVersion}/kafka_${version}.tgz";
+    inherit hash;
+  };
+
+  nativeBuildInputs = [ makeWrapper ];
+  buildInputs = [
+    jre
+    bash
+    gnugrep
+    gnused
+    coreutils
+    ps
+  ];
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R config libs $out
+
+    mkdir -p $out/bin
+    cp bin/kafka* $out/bin
+    cp bin/connect* $out/bin
+
+    # allow us the specify logging directory using env
+    substituteInPlace $out/bin/kafka-run-class.sh \
+      --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"'
+
+    substituteInPlace $out/bin/kafka-server-stop.sh \
+      --replace 'ps' '${ps}/bin/ps'
+
+    for p in $out/bin\/*.sh; do
+      wrapProgram $p \
+        --set JAVA_HOME "${jre}" \
+        --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \
+        --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin"
+    done
+    chmod +x $out/bin\/*
+  '';
+
+  passthru = {
+    inherit jre; # Used by the NixOS module to select the supported JRE
+    tests.nixos = nixosTests.kafka.base.${"kafka_" + series};
+    updateScript = [
+      ./update.sh
+      seriesFile
+    ];
+  };
+
+  meta = {
+    homepage = "https://kafka.apache.org";
+    description = "High-throughput distributed messaging system";
+    license = lib.licenses.asl20;
+    sourceProvenance = [ lib.sourceTypes.binaryBytecode ];
+    maintainers = [ lib.maintainers.ragge ];
+    platforms = lib.platforms.unix;
+  };
+}

pkgs/servers/apache-kafka/update.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p curl jq nix
+
+set -euo pipefail
+echoerr() { echo "$@" 1>&2; }
+
+fname="$1"
+echoerr "Working on $fname"
+
+# Extract series from filename: 4_1.nix → 4.1
+series=$(basename "$fname" .nix | tr '_' '.')
+echoerr "Series: $series"
+
+# Read current values from the nix file.
+current_kafka_version=$(sed -n 's/.*kafkaVersion = "\(.*\)";/\1/p' "$fname")
+scala_version=$(sed -n 's/.*scalaVersion = "\(.*\)";/\1/p' "$fname")
+echoerr "Current: kafkaVersion=$current_kafka_version scalaVersion=$scala_version"
+
+# Build a regex that matches e.g. "4.1.0", "4.1.1", but not "4.1.0-rc1".
+series_re="^$(echo "$series" | sed 's/\./\\./g')\\.[0-9]+$"
+
+# Fetch tags from the Apache Kafka GitHub repo.
+# Tags are plain semver (no "v" prefix); filter out release candidates.
+latest_version=$(
+  { for page in 1 2 3; do
+      curl -fsSL ${GITHUB_TOKEN:+-u ":${GITHUB_TOKEN}"} \
+        "https://api.github.com/repos/apache/kafka/tags?per_page=100&page=$page"
+    done; } \
+  | jq -r '.[].name' \
+  | grep -E "$series_re" \
+  | sort -V \
+  | tail -1
+)
+
+if [[ -z "$latest_version" ]]; then
+  echoerr "Error: no tags found for series $series"
+  exit 1
+fi
+
+echoerr "Latest version for series $series: $latest_version"
+
+if [[ "$latest_version" == "$current_kafka_version" ]]; then
+  echoerr "Already at latest version $latest_version, nothing to do"
+  exit 0
+fi
+
+# Update kafkaVersion in the nix file.
+sed -i -E "s|(kafkaVersion = \").*(\";)|\1$latest_version\2|" "$fname"
+grep -q "kafkaVersion = \"$latest_version\"" "$fname"
+
+# Compute new SRI hash for the Apache mirror tarball.
+tarball_url="https://archive.apache.org/dist/kafka/${latest_version}/kafka_${scala_version}-${latest_version}.tgz"
+echoerr "Fetching hash for: $tarball_url"
+
+new_hash=$(nix-prefetch-url --type sha256 "$tarball_url" 2>/dev/null)
+if [[ -z "$new_hash" ]]; then
+  echoerr "Error: failed to fetch hash for $tarball_url"
+  exit 1
+fi
+new_hash=$(nix hash convert --hash-algo sha256 --to sri "$new_hash")
+
+echoerr "New hash: $new_hash"
+sed -i -E "s|(hash = \").*(\";)|\1$new_hash\2|" "$fname"
+grep -q "$new_hash" "$fname"
+
+echoerr "Done. Updated $fname from $current_kafka_version to $latest_version"