From d75b01eb76f4c98423ebda6c49f6b8a08570f211 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 12 Mar 2026 22:12:34 +0000 Subject: [PATCH 1/2] apacheKafka: Refactor into per-series files with generic.nix Split the monolithic default.nix versionMap into per-series files that import a shared generic.nix --- pkgs/servers/apache-kafka/3_9.nix | 6 ++ pkgs/servers/apache-kafka/4_0.nix | 6 ++ pkgs/servers/apache-kafka/4_1.nix | 6 ++ pkgs/servers/apache-kafka/default.nix | 106 ++------------------------ pkgs/servers/apache-kafka/generic.nix | 82 ++++++++++++++++++++ 5 files changed, 105 insertions(+), 101 deletions(-) create mode 100644 pkgs/servers/apache-kafka/3_9.nix create mode 100644 pkgs/servers/apache-kafka/4_0.nix create mode 100644 pkgs/servers/apache-kafka/4_1.nix create mode 100644 pkgs/servers/apache-kafka/generic.nix diff --git a/pkgs/servers/apache-kafka/3_9.nix b/pkgs/servers/apache-kafka/3_9.nix new file mode 100644 index 000000000000..27fa276fb09f --- /dev/null +++ b/pkgs/servers/apache-kafka/3_9.nix @@ -0,0 +1,6 @@ +import ./generic.nix { + seriesFile = ./3_9.nix; + kafkaVersion = "3.9.1"; + scalaVersion = "2.13"; + sha256 = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; +} diff --git a/pkgs/servers/apache-kafka/4_0.nix b/pkgs/servers/apache-kafka/4_0.nix new file mode 100644 index 000000000000..1b87feaa230f --- /dev/null +++ b/pkgs/servers/apache-kafka/4_0.nix @@ -0,0 +1,6 @@ +import ./generic.nix { + seriesFile = ./4_0.nix; + kafkaVersion = "4.0.1"; + scalaVersion = "2.13"; + sha256 = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; +} diff --git a/pkgs/servers/apache-kafka/4_1.nix b/pkgs/servers/apache-kafka/4_1.nix new file mode 100644 index 000000000000..4c226ec3d6d7 --- /dev/null +++ b/pkgs/servers/apache-kafka/4_1.nix @@ -0,0 +1,6 @@ +import ./generic.nix { + seriesFile = ./4_1.nix; + kafkaVersion = "4.1.1"; + scalaVersion = "2.13"; + sha256 = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; +} diff --git a/pkgs/servers/apache-kafka/default.nix b/pkgs/servers/apache-kafka/default.nix index d6eacbe3d130..039c829f9bbc 100644 --- a/pkgs/servers/apache-kafka/default.nix +++ b/pkgs/servers/apache-kafka/default.nix @@ -1,102 +1,6 @@ +{ callPackage }: { - lib, - stdenv, - fetchurl, - jdk17_headless, - makeWrapper, - bash, - coreutils, - gnugrep, - gnused, - ps, - nixosTests, -}: - -let - versionMap = { - "4_1" = { - kafkaVersion = "4.1.1"; - scalaVersion = "2.13"; - sha256 = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_4_1; - }; - "4_0" = { - kafkaVersion = "4.0.1"; - scalaVersion = "2.13"; - sha256 = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_4_0; - }; - "3_9" = { - kafkaVersion = "3.9.1"; - scalaVersion = "2.13"; - sha256 = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_3_9; - }; - }; - - build = - versionInfo: - stdenv.mkDerivation rec { - version = "${versionInfo.scalaVersion}-${versionInfo.kafkaVersion}"; - pname = "apache-kafka"; - - src = fetchurl { - url = "mirror://apache/kafka/${versionInfo.kafkaVersion}/kafka_${version}.tgz"; - inherit (versionInfo) sha256; - }; - - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ - versionInfo.jre - bash - gnugrep - gnused - coreutils - ps - ]; - - installPhase = '' - mkdir -p $out - cp -R config libs $out - - mkdir -p $out/bin - cp bin/kafka* $out/bin - cp bin/connect* $out/bin - - # allow us the specify logging directory using env - substituteInPlace $out/bin/kafka-run-class.sh \ - --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"' - - substituteInPlace $out/bin/kafka-server-stop.sh \ - --replace 'ps' '${ps}/bin/ps' - - for p in $out/bin\/*.sh; do - wrapProgram $p \ - --set JAVA_HOME "${versionInfo.jre}" \ - --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \ - --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin" - done - chmod +x $out/bin\/* - ''; - - passthru = { - inherit (versionInfo) jre; # Used by the NixOS module to select the supported JRE - tests.nixos = versionInfo.nixosTest; - }; - - meta = { - homepage = "https://kafka.apache.org"; - description = "High-throughput distributed messaging system"; - license = lib.licenses.asl20; - sourceProvenance = [ lib.sourceTypes.binaryBytecode ]; - maintainers = [ lib.maintainers.ragge ]; - platforms = lib.platforms.unix; - }; - }; -in -lib.mapAttrs' ( - majorVersion: versionInfo: lib.nameValuePair "apacheKafka_${majorVersion}" (build versionInfo) -) versionMap + apacheKafka_4_1 = callPackage ./4_1.nix { }; + apacheKafka_4_0 = callPackage ./4_0.nix { }; + apacheKafka_3_9 = callPackage ./3_9.nix { }; +} diff --git a/pkgs/servers/apache-kafka/generic.nix b/pkgs/servers/apache-kafka/generic.nix new file mode 100644 index 000000000000..1d52bd5e73d2 --- /dev/null +++ b/pkgs/servers/apache-kafka/generic.nix @@ -0,0 +1,82 @@ +{ + seriesFile, + kafkaVersion, + scalaVersion, + sha256, +}: + +{ + lib, + stdenv, + fetchurl, + jdk17_headless, + makeWrapper, + bash, + coreutils, + gnugrep, + gnused, + ps, + nixosTests, +}: + +let + jre = jdk17_headless; + series = lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kafkaVersion); +in +stdenv.mkDerivation rec { + version = "${scalaVersion}-${kafkaVersion}"; + pname = "apache-kafka"; + + src = fetchurl { + url = "mirror://apache/kafka/${kafkaVersion}/kafka_${version}.tgz"; + inherit sha256; + }; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ + jre + bash + gnugrep + gnused + coreutils + ps + ]; + + installPhase = '' + mkdir -p $out + cp -R config libs $out + + mkdir -p $out/bin + cp bin/kafka* $out/bin + cp bin/connect* $out/bin + + # allow us the specify logging directory using env + substituteInPlace $out/bin/kafka-run-class.sh \ + --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"' + + substituteInPlace $out/bin/kafka-server-stop.sh \ + --replace 'ps' '${ps}/bin/ps' + + for p in $out/bin\/*.sh; do + wrapProgram $p \ + --set JAVA_HOME "${jre}" \ + --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \ + --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin" + done + chmod +x $out/bin\/* + ''; + + passthru = { + inherit jre; # Used by the NixOS module to select the supported JRE + tests.nixos = nixosTests.kafka.base.${"kafka_" + series}; + }; + + meta = { + homepage = "https://kafka.apache.org"; + description = "High-throughput distributed messaging system"; + license = lib.licenses.asl20; + sourceProvenance = [ lib.sourceTypes.binaryBytecode ]; + maintainers = [ lib.maintainers.ragge ]; + platforms = lib.platforms.unix; + }; +} From d2b2dd07efdaae55bac2c9f176e55077947fef2d Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 12 Mar 2026 22:12:54 +0000 Subject: [PATCH 2/2] apacheKafka: Add update script for per-series version updates --- pkgs/servers/apache-kafka/3_9.nix | 3 +- pkgs/servers/apache-kafka/4_0.nix | 3 +- pkgs/servers/apache-kafka/4_1.nix | 3 +- pkgs/servers/apache-kafka/generic.nix | 10 ++-- pkgs/servers/apache-kafka/update.sh | 66 +++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 9 deletions(-) create mode 100755 pkgs/servers/apache-kafka/update.sh diff --git a/pkgs/servers/apache-kafka/3_9.nix b/pkgs/servers/apache-kafka/3_9.nix index 27fa276fb09f..ecd0ed75f784 100644 --- a/pkgs/servers/apache-kafka/3_9.nix +++ b/pkgs/servers/apache-kafka/3_9.nix @@ -1,6 +1,5 @@ import ./generic.nix { - seriesFile = ./3_9.nix; kafkaVersion = "3.9.1"; scalaVersion = "2.13"; - sha256 = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; + hash = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; } diff --git a/pkgs/servers/apache-kafka/4_0.nix b/pkgs/servers/apache-kafka/4_0.nix index 1b87feaa230f..a35c9610c12b 100644 --- a/pkgs/servers/apache-kafka/4_0.nix +++ b/pkgs/servers/apache-kafka/4_0.nix @@ -1,6 +1,5 @@ import ./generic.nix { - seriesFile = ./4_0.nix; kafkaVersion = "4.0.1"; scalaVersion = "2.13"; - sha256 = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; + hash = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; } diff --git a/pkgs/servers/apache-kafka/4_1.nix b/pkgs/servers/apache-kafka/4_1.nix index 4c226ec3d6d7..933054416127 100644 --- a/pkgs/servers/apache-kafka/4_1.nix +++ b/pkgs/servers/apache-kafka/4_1.nix @@ -1,6 +1,5 @@ import ./generic.nix { - seriesFile = ./4_1.nix; kafkaVersion = "4.1.1"; scalaVersion = "2.13"; - sha256 = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; + hash = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; } diff --git a/pkgs/servers/apache-kafka/generic.nix b/pkgs/servers/apache-kafka/generic.nix index 1d52bd5e73d2..caed500306f3 100644 --- a/pkgs/servers/apache-kafka/generic.nix +++ b/pkgs/servers/apache-kafka/generic.nix @@ -1,8 +1,7 @@ { - seriesFile, kafkaVersion, scalaVersion, - sha256, + hash, }: { @@ -22,6 +21,7 @@ let jre = jdk17_headless; series = lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kafkaVersion); + seriesFile = ./. + "/${series}.nix"; in stdenv.mkDerivation rec { version = "${scalaVersion}-${kafkaVersion}"; @@ -29,7 +29,7 @@ stdenv.mkDerivation rec { src = fetchurl { url = "mirror://apache/kafka/${kafkaVersion}/kafka_${version}.tgz"; - inherit sha256; + inherit hash; }; nativeBuildInputs = [ makeWrapper ]; @@ -69,6 +69,10 @@ stdenv.mkDerivation rec { passthru = { inherit jre; # Used by the NixOS module to select the supported JRE tests.nixos = nixosTests.kafka.base.${"kafka_" + series}; + updateScript = [ + ./update.sh + seriesFile + ]; }; meta = { diff --git a/pkgs/servers/apache-kafka/update.sh b/pkgs/servers/apache-kafka/update.sh new file mode 100755 index 000000000000..37fd33e5ae6d --- /dev/null +++ b/pkgs/servers/apache-kafka/update.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl jq nix + +set -euo pipefail +echoerr() { echo "$@" 1>&2; } + +fname="$1" +echoerr "Working on $fname" + +# Extract series from filename: 4_1.nix → 4.1 +series=$(basename "$fname" .nix | tr '_' '.') +echoerr "Series: $series" + +# Read current values from the nix file. +current_kafka_version=$(sed -n 's/.*kafkaVersion = "\(.*\)";/\1/p' "$fname") +scala_version=$(sed -n 's/.*scalaVersion = "\(.*\)";/\1/p' "$fname") +echoerr "Current: kafkaVersion=$current_kafka_version scalaVersion=$scala_version" + +# Build a regex that matches e.g. "4.1.0", "4.1.1", but not "4.1.0-rc1". +series_re="^$(echo "$series" | sed 's/\./\\./g')\\.[0-9]+$" + +# Fetch tags from the Apache Kafka GitHub repo. +# Tags are plain semver (no "v" prefix); filter out release candidates. +latest_version=$( + { for page in 1 2 3; do + curl -fsSL ${GITHUB_TOKEN:+-u ":${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/apache/kafka/tags?per_page=100&page=$page" + done; } \ + | jq -r '.[].name' \ + | grep -E "$series_re" \ + | sort -V \ + | tail -1 +) + +if [[ -z "$latest_version" ]]; then + echoerr "Error: no tags found for series $series" + exit 1 +fi + +echoerr "Latest version for series $series: $latest_version" + +if [[ "$latest_version" == "$current_kafka_version" ]]; then + echoerr "Already at latest version $latest_version, nothing to do" + exit 0 +fi + +# Update kafkaVersion in the nix file. +sed -i -E "s|(kafkaVersion = \").*(\";)|\1$latest_version\2|" "$fname" +grep -q "kafkaVersion = \"$latest_version\"" "$fname" + +# Compute new SRI hash for the Apache mirror tarball. +tarball_url="https://archive.apache.org/dist/kafka/${latest_version}/kafka_${scala_version}-${latest_version}.tgz" +echoerr "Fetching hash for: $tarball_url" + +new_hash=$(nix-prefetch-url --type sha256 "$tarball_url" 2>/dev/null) +if [[ -z "$new_hash" ]]; then + echoerr "Error: failed to fetch hash for $tarball_url" + exit 1 +fi +new_hash=$(nix hash convert --hash-algo sha256 --to sri "$new_hash") + +echoerr "New hash: $new_hash" +sed -i -E "s|(hash = \").*(\";)|\1$new_hash\2|" "$fname" +grep -q "$new_hash" "$fname" + +echoerr "Done. Updated $fname from $current_kafka_version to $latest_version"