diff --git a/pkgs/servers/apache-kafka/3_9.nix b/pkgs/servers/apache-kafka/3_9.nix new file mode 100644 index 000000000000..ecd0ed75f784 --- /dev/null +++ b/pkgs/servers/apache-kafka/3_9.nix @@ -0,0 +1,5 @@ +import ./generic.nix { + kafkaVersion = "3.9.1"; + scalaVersion = "2.13"; + hash = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; +} diff --git a/pkgs/servers/apache-kafka/4_0.nix b/pkgs/servers/apache-kafka/4_0.nix new file mode 100644 index 000000000000..a35c9610c12b --- /dev/null +++ b/pkgs/servers/apache-kafka/4_0.nix @@ -0,0 +1,5 @@ +import ./generic.nix { + kafkaVersion = "4.0.1"; + scalaVersion = "2.13"; + hash = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; +} diff --git a/pkgs/servers/apache-kafka/4_1.nix b/pkgs/servers/apache-kafka/4_1.nix new file mode 100644 index 000000000000..933054416127 --- /dev/null +++ b/pkgs/servers/apache-kafka/4_1.nix @@ -0,0 +1,5 @@ +import ./generic.nix { + kafkaVersion = "4.1.1"; + scalaVersion = "2.13"; + hash = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; +} diff --git a/pkgs/servers/apache-kafka/default.nix b/pkgs/servers/apache-kafka/default.nix index d6eacbe3d130..039c829f9bbc 100644 --- a/pkgs/servers/apache-kafka/default.nix +++ b/pkgs/servers/apache-kafka/default.nix @@ -1,102 +1,6 @@ +{ callPackage }: { - lib, - stdenv, - fetchurl, - jdk17_headless, - makeWrapper, - bash, - coreutils, - gnugrep, - gnused, - ps, - nixosTests, -}: - -let - versionMap = { - "4_1" = { - kafkaVersion = "4.1.1"; - scalaVersion = "2.13"; - sha256 = "sha256-eR6O5plpgtgFWCk2eoA5H5TBvKcymy+7ZYv8IRp3Ry4="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_4_1; - }; - "4_0" = { - kafkaVersion = "4.0.1"; - scalaVersion = "2.13"; - sha256 = "sha256-M8xc04WE6w9yYWxMLwX7SINKVvoKnNn+AKxzIMNM9SQ="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_4_0; - }; - "3_9" = { - kafkaVersion = "3.9.1"; - scalaVersion = "2.13"; - sha256 = "sha256-3UOZgW5niUbKt2470WhhA1VeabyPKrhobNpxqhW8MaM="; - jre = jdk17_headless; - nixosTest = nixosTests.kafka.base.kafka_3_9; - }; - }; - - build = - versionInfo: - stdenv.mkDerivation rec { - version = "${versionInfo.scalaVersion}-${versionInfo.kafkaVersion}"; - pname = "apache-kafka"; - - src = fetchurl { - url = "mirror://apache/kafka/${versionInfo.kafkaVersion}/kafka_${version}.tgz"; - inherit (versionInfo) sha256; - }; - - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ - versionInfo.jre - bash - gnugrep - gnused - coreutils - ps - ]; - - installPhase = '' - mkdir -p $out - cp -R config libs $out - - mkdir -p $out/bin - cp bin/kafka* $out/bin - cp bin/connect* $out/bin - - # allow us the specify logging directory using env - substituteInPlace $out/bin/kafka-run-class.sh \ - --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"' - - substituteInPlace $out/bin/kafka-server-stop.sh \ - --replace 'ps' '${ps}/bin/ps' - - for p in $out/bin\/*.sh; do - wrapProgram $p \ - --set JAVA_HOME "${versionInfo.jre}" \ - --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \ - --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin" - done - chmod +x $out/bin\/* - ''; - - passthru = { - inherit (versionInfo) jre; # Used by the NixOS module to select the supported JRE - tests.nixos = versionInfo.nixosTest; - }; - - meta = { - homepage = "https://kafka.apache.org"; - description = "High-throughput distributed messaging system"; - license = lib.licenses.asl20; - sourceProvenance = [ lib.sourceTypes.binaryBytecode ]; - maintainers = [ lib.maintainers.ragge ]; - platforms = lib.platforms.unix; - }; - }; -in -lib.mapAttrs' ( - majorVersion: versionInfo: lib.nameValuePair "apacheKafka_${majorVersion}" (build versionInfo) -) versionMap + apacheKafka_4_1 = callPackage ./4_1.nix { }; + apacheKafka_4_0 = callPackage ./4_0.nix { }; + apacheKafka_3_9 = callPackage ./3_9.nix { }; +} diff --git a/pkgs/servers/apache-kafka/generic.nix b/pkgs/servers/apache-kafka/generic.nix new file mode 100644 index 000000000000..caed500306f3 --- /dev/null +++ b/pkgs/servers/apache-kafka/generic.nix @@ -0,0 +1,86 @@ +{ + kafkaVersion, + scalaVersion, + hash, +}: + +{ + lib, + stdenv, + fetchurl, + jdk17_headless, + makeWrapper, + bash, + coreutils, + gnugrep, + gnused, + ps, + nixosTests, +}: + +let + jre = jdk17_headless; + series = lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kafkaVersion); + seriesFile = ./. + "/${series}.nix"; +in +stdenv.mkDerivation rec { + version = "${scalaVersion}-${kafkaVersion}"; + pname = "apache-kafka"; + + src = fetchurl { + url = "mirror://apache/kafka/${kafkaVersion}/kafka_${version}.tgz"; + inherit hash; + }; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ + jre + bash + gnugrep + gnused + coreutils + ps + ]; + + installPhase = '' + mkdir -p $out + cp -R config libs $out + + mkdir -p $out/bin + cp bin/kafka* $out/bin + cp bin/connect* $out/bin + + # allow us the specify logging directory using env + substituteInPlace $out/bin/kafka-run-class.sh \ + --replace 'LOG_DIR="$base_dir/logs"' 'LOG_DIR="$KAFKA_LOG_DIR"' + + substituteInPlace $out/bin/kafka-server-stop.sh \ + --replace 'ps' '${ps}/bin/ps' + + for p in $out/bin\/*.sh; do + wrapProgram $p \ + --set JAVA_HOME "${jre}" \ + --set KAFKA_LOG_DIR "/tmp/apache-kafka-logs" \ + --prefix PATH : "${bash}/bin:${coreutils}/bin:${gnugrep}/bin:${gnused}/bin" + done + chmod +x $out/bin\/* + ''; + + passthru = { + inherit jre; # Used by the NixOS module to select the supported JRE + tests.nixos = nixosTests.kafka.base.${"kafka_" + series}; + updateScript = [ + ./update.sh + seriesFile + ]; + }; + + meta = { + homepage = "https://kafka.apache.org"; + description = "High-throughput distributed messaging system"; + license = lib.licenses.asl20; + sourceProvenance = [ lib.sourceTypes.binaryBytecode ]; + maintainers = [ lib.maintainers.ragge ]; + platforms = lib.platforms.unix; + }; +} diff --git a/pkgs/servers/apache-kafka/update.sh b/pkgs/servers/apache-kafka/update.sh new file mode 100755 index 000000000000..37fd33e5ae6d --- /dev/null +++ b/pkgs/servers/apache-kafka/update.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl jq nix + +set -euo pipefail +echoerr() { echo "$@" 1>&2; } + +fname="$1" +echoerr "Working on $fname" + +# Extract series from filename: 4_1.nix → 4.1 +series=$(basename "$fname" .nix | tr '_' '.') +echoerr "Series: $series" + +# Read current values from the nix file. +current_kafka_version=$(sed -n 's/.*kafkaVersion = "\(.*\)";/\1/p' "$fname") +scala_version=$(sed -n 's/.*scalaVersion = "\(.*\)";/\1/p' "$fname") +echoerr "Current: kafkaVersion=$current_kafka_version scalaVersion=$scala_version" + +# Build a regex that matches e.g. "4.1.0", "4.1.1", but not "4.1.0-rc1". +series_re="^$(echo "$series" | sed 's/\./\\./g')\\.[0-9]+$" + +# Fetch tags from the Apache Kafka GitHub repo. +# Tags are plain semver (no "v" prefix); filter out release candidates. +latest_version=$( + { for page in 1 2 3; do + curl -fsSL ${GITHUB_TOKEN:+-u ":${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/apache/kafka/tags?per_page=100&page=$page" + done; } \ + | jq -r '.[].name' \ + | grep -E "$series_re" \ + | sort -V \ + | tail -1 +) + +if [[ -z "$latest_version" ]]; then + echoerr "Error: no tags found for series $series" + exit 1 +fi + +echoerr "Latest version for series $series: $latest_version" + +if [[ "$latest_version" == "$current_kafka_version" ]]; then + echoerr "Already at latest version $latest_version, nothing to do" + exit 0 +fi + +# Update kafkaVersion in the nix file. +sed -i -E "s|(kafkaVersion = \").*(\";)|\1$latest_version\2|" "$fname" +grep -q "kafkaVersion = \"$latest_version\"" "$fname" + +# Compute new SRI hash for the Apache mirror tarball. +tarball_url="https://archive.apache.org/dist/kafka/${latest_version}/kafka_${scala_version}-${latest_version}.tgz" +echoerr "Fetching hash for: $tarball_url" + +new_hash=$(nix-prefetch-url --type sha256 "$tarball_url" 2>/dev/null) +if [[ -z "$new_hash" ]]; then + echoerr "Error: failed to fetch hash for $tarball_url" + exit 1 +fi +new_hash=$(nix hash convert --hash-algo sha256 --to sri "$new_hash") + +echoerr "New hash: $new_hash" +sed -i -E "s|(hash = \").*(\";)|\1$new_hash\2|" "$fname" +grep -q "$new_hash" "$fname" + +echoerr "Done. Updated $fname from $current_kafka_version to $latest_version"