python3Packages.semgrep: 1.164.0 -> 1.172.0

Diff: https://github.com/semgrep/semgrep/compare/v1.164.0...v1.172.0

Changelog: https://github.com/semgrep/semgrep/blob/v1.172.0/CHANGELOG.md

pkgs/development/python-modules/semgrep/common.nix

@@ -1,9 +1,9 @@
 { lib }:
 
 rec {
-  version = "1.164.0";
+  version = "1.172.0";
 
-  srcHash = "sha256-ced287/jH+as/1rGBOfoZ06UuQ1sf1YI4AMHbHrtnHU=";
+  srcHash = "sha256-dILO0ia4zriHiC1jVv02YOyj0Snni4aY66g+omApqSQ=";
 
   # This tag is used to select the correct wheel from PyPI.
   # It is updated by the update.sh script.
@@ -17,8 +17,8 @@ rec {
     "cli/src/semgrep/semgrep_interfaces" = {
       owner = "semgrep";
       repo = "semgrep-interfaces";
-      rev = "f4a74a03e8ec3dd368b96101648a3210e03fa61e";
-      hash = "sha256-dy+oOB0QmZjMpTYINSPIjzhpN6d/45DaajqumKIYxC4=";
+      rev = "6dc898658d554ce80e6fdd58904adea2fd0e30c8";
+      hash = "sha256-7JMo2TU5JbPscrfI1qdz1P2bF6J8dTDhOqAQXxa3tm8=";
     };
   };
 
@@ -29,15 +29,15 @@ rec {
   core = {
     x86_64-linux = {
       platform = "manylinux_2_34_x86_64";
-      hash = "sha256-dFrlzhvvfJsDyStDHRdMpu54AaXioEfGSsIQTH5pUvs=";
+      hash = "sha256-2LlK9CZqV1KHrSzYRFc3Q6tP5Y9r+22SKTJ4B5N+reM=";
     };
     aarch64-linux = {
       platform = "manylinux_2_34_aarch64";
-      hash = "sha256-N24E9xOyRO7pXopRs+gSQM2nwHE214GfcntcoH7H7Kk=";
+      hash = "sha256-yIGjBbll5ZS4ixXCxkGbOY525DjsYeYJFsiy7/6SckA=";
     };
     aarch64-darwin = {
       platform = "macosx_11_0_arm64";
-      hash = "sha256-AsKxA5Wmy3NEQJ0kS6ylE33d0W86e9F494aiIkwyrcA=";
+      hash = "sha256-CeksnmwWNaFUnU4pey1KloTc01GyFu1/LrUc/FVHnEk=";
     };
   };
 

pkgs/development/python-modules/semgrep/default.nix

@@ -5,8 +5,10 @@
   semgrep-core,
 
   # check tools
-  git,
+  addBinToPathHook,
+  gitMinimal,
   pytestCheckHook,
+  writableTmpDirAsHomeHook,
 
   # python runtime dependencies
   attrs,
@@ -15,6 +17,7 @@
   click-option-group,
   colorama,
   defusedxml,
+  exceptiongroup,
   glom,
   jsonschema,
   mcp,
@@ -51,33 +54,54 @@
 
 let
   common = import ./common.nix { inherit lib; };
-  semgrepBinPath = lib.makeBinPath [ semgrep-core ];
 in
-buildPythonPackage rec {
-  format = "setuptools";
+buildPythonPackage (finalAttrs: {
   pname = "semgrep";
   inherit (common) version;
+  pyproject = true;
+  __structuredAttrs = true;
+
   src = fetchFromGitHub {
     owner = "semgrep";
     repo = "semgrep";
-    rev = "v${version}";
+    tag = "v${finalAttrs.version}";
     hash = common.srcHash;
   };
 
+  sourceRoot = "${finalAttrs.src.name}/cli";
+
   # prepare a subset of the submodules as we only need a handful
   # and there are many many submodules total
   postPatch =
-    (lib.concatStringsSep "\n" (
-      lib.mapAttrsToList (path: submodule: ''
-        # substitute ${path}
-        # remove git submodule placeholder
-        rm -r ${path}
-        # link submodule
-        ln -s ${submodule}/ ${path}
-      '') passthru.submodulesSubset
-    ))
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (
+        path: submodule:
+        let
+          path' = lib.removePrefix "cli/" path;
+        in
+        ''
+          # substitute ${path'}
+          # remove git submodule placeholder
+          rm -r ${path'}
+          # link submodule
+          ln -s ${submodule}/ ${path'}
+        ''
+      ) finalAttrs.passthru.submodulesSubset
+    )
+    # hardcode the path to the shared `semgrep-core` binary so that it does not
+    # need to be copied into the wheel nor be present on PATH at runtime.
+    # There are two independent lookups: one in the library and one in the CLI
+    # entrypoint (which execs semgrep-core directly).
     + ''
-      cd cli
+      substituteInPlace src/semgrep/semgrep_core.py \
+        --replace-fail \
+          'ret = compute_executable_path("semgrep-core")' \
+          'ret = "${lib.getExe semgrep-core}"'
+
+      substituteInPlace src/semgrep/console_scripts/entrypoint.py \
+        --replace-fail \
+          'path = shutil.which(core)' \
+          'path = "${lib.getExe semgrep-core}" if not pro else shutil.which(core)'
     '';
 
   # tell cli/setup.py to not copy semgrep-core into the result
@@ -87,9 +111,16 @@ buildPythonPackage rec {
 
   pythonRelaxDeps = [
     "boltons"
+    "click"
+    "exceptiongroup"
     "glom"
+    "jsonschema"
+    "mcp"
+    "opentelemetry-api"
+    "opentelemetry-exporter-otlp-proto-http"
+    "opentelemetry-sdk"
+    "wcmatch"
   ];
-
   dependencies = [
     attrs
     boltons
@@ -97,6 +128,7 @@ buildPythonPackage rec {
     click-option-group
     colorama
     defusedxml
+    exceptiongroup
     glom
     jsonschema
     mcp
@@ -119,11 +151,11 @@ buildPythonPackage rec {
     wcmatch
   ];
 
-  doCheck = true;
-
   nativeCheckInputs = [
-    git
+    addBinToPathHook
+    gitMinimal
     pytestCheckHook
+    writableTmpDirAsHomeHook
 
     flaky
     pytest-asyncio
@@ -152,35 +184,18 @@ buildPythonPackage rec {
     "test_send"
     # many child tests require networking to download files
     "TestConfigLoaderForProducts"
+    # require networking (pro install metrics are sent to semgrep.dev)
+    "test_install_command_download_error_records_download_reason"
+    "test_install_command_metrics_off"
+    "test_install_command_sends_failure_metrics"
+    "test_install_command_sends_metrics_when_logged_in"
+    "test_run_install_success_records_outcome"
+    "test_run_install_version_check_failure_records_error"
   ];
 
-  preCheck = ''
-    # tests need a home directory
-    export HOME="$(mktemp -d)"
-
-    # tests need access to `semgrep-core`
-    export OLD_PATH="$PATH"
-    export PATH="$PATH:${semgrepBinPath}"
-  '';
-
-  postCheck = ''
-    export PATH="$OLD_PATH"
-    unset OLD_PATH
-  '';
-
-  # since we stop cli/setup.py from finding semgrep-core and copying it into
-  # the result we need to provide it on the PATH
-  preFixup = ''
-    makeWrapperArgs+=(--prefix PATH : ${semgrepBinPath})
-  '';
-
-  postInstall = ''
-    chmod +x $out/bin/{,py}semgrep
-  '';
-
   passthru = {
     inherit common semgrep-core;
-    submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules;
+    submodulesSubset = lib.mapAttrs (_: args: fetchFromGitHub args) common.submodules;
     updateScript = ./update.sh;
   };
 
@@ -188,4 +203,4 @@ buildPythonPackage rec {
     description = common.meta.description + " - cli";
     inherit (semgrep-core.meta) platforms;
   };
-}
+})

pkgs/development/python-modules/semgrep/semgrep-core.nix

@@ -3,17 +3,20 @@
   stdenv,
   fetchPypi,
 
-  autoPatchelfHook,
+  patchelf,
   unzip,
 }:
 
 let
   common = import ./common.nix { inherit lib; };
 in
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "semgrep-core";
   inherit (common) version;
 
+  __structuredAttrs = true;
+  strictDeps = true;
+
   # fetch pre-built semgrep-core since the ocaml build is complex and relies on
   # the opam package manager at some point
   # pulling it out of the python wheel as r2c no longer release a built binary
@@ -22,22 +25,23 @@ stdenv.mkDerivation rec {
     let
       inherit (stdenv.hostPlatform) system;
       data = common.core.${system} or (throw "Unsupported system: ${system}");
+      python = common.pythonWheelTag;
     in
-    fetchPypi rec {
+    fetchPypi {
       pname = "semgrep";
-      inherit version;
+      inherit (finalAttrs) version;
       format = "wheel";
       dist = python;
-      python = common.pythonWheelTag;
+      inherit python;
       inherit (data) platform hash;
     };
 
   nativeBuildInputs = [
     unzip
   ]
-  ++ lib.optional stdenv.hostPlatform.isLinux autoPatchelfHook;
-
-  buildInputs = lib.optional stdenv.hostPlatform.isLinux stdenv.cc.cc.lib;
+  ++ lib.optionals stdenv.hostPlatform.isLinux [
+    patchelf
+  ];
 
   # _tryUnzip from unzip's setup-hook doesn't recognise .whl
   # "do not know how to unpack source archive"
@@ -53,26 +57,36 @@ stdenv.mkDerivation rec {
 
   installPhase = ''
     runHook preInstall
-    install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core
+    install -Dm 755 -t $out/bin semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/semgrep-core
 
     # copy bundled libs as well
     # keeping them in bin/libs matches the layout in the wheel
-    if [ -d semgrep-${version}.data/purelib/semgrep/bin/libs ]; then
+    if [ -d semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/libs ]; then
       mkdir -p $out/bin/libs
-      cp -rf semgrep-${version}.data/purelib/semgrep/bin/libs/* $out/bin/libs/
-
-      # help autoPatchelfHook find these libs
-      if [ -n "''${autoPatchelfHook:-}" ]; then
-        appendAutoPatchelfSearchPath $out/bin/libs
-      fi
+      cp -rf semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/libs/* $out/bin/libs/
     fi
     runHook postInstall
   '';
 
+  # Multiple rewrites of this large binary corrupt it on aarch64: patchelf leaves `.dynstr`
+  # outside any loadable segment, so ld.so segfaults reading the rpath at startup (patchelf bug,
+  # see https://github.com/NixOS/patchelf/issues/244). So we avoid autoPatchelfHook, disable
+  # stdenv's own `strip` and `patchelf --shrink-rpath`, and fix the interpreter and rpath in a
+  # single patchelf call below. The bundled bin/libs reference each other via $ORIGIN, so only the
+  # main binary needs it.
+  dontPatchELF = true;
+  dontStrip = true;
+  postFixup = lib.optionalString stdenv.hostPlatform.isLinux ''
+    patchelf \
+      --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \
+      --set-rpath "$out/bin/libs:${lib.makeLibraryPath [ stdenv.cc.libc ]}" \
+      $out/bin/semgrep-core
+  '';
+
   meta = common.meta // {
     description = common.meta.description + " - core binary";
     mainProgram = "semgrep-core";
     sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];
     platforms = lib.attrNames common.core;
   };
-}
+})