diff --git a/pkgs/development/python-modules/semgrep/common.nix b/pkgs/development/python-modules/semgrep/common.nix index e6343b7edba9..cd193e883fbe 100644 --- a/pkgs/development/python-modules/semgrep/common.nix +++ b/pkgs/development/python-modules/semgrep/common.nix @@ -1,9 +1,9 @@ { lib }: rec { - version = "1.164.0"; + version = "1.172.0"; - srcHash = "sha256-ced287/jH+as/1rGBOfoZ06UuQ1sf1YI4AMHbHrtnHU="; + srcHash = "sha256-dILO0ia4zriHiC1jVv02YOyj0Snni4aY66g+omApqSQ="; # This tag is used to select the correct wheel from PyPI. # It is updated by the update.sh script. @@ -17,8 +17,8 @@ rec { "cli/src/semgrep/semgrep_interfaces" = { owner = "semgrep"; repo = "semgrep-interfaces"; - rev = "f4a74a03e8ec3dd368b96101648a3210e03fa61e"; - hash = "sha256-dy+oOB0QmZjMpTYINSPIjzhpN6d/45DaajqumKIYxC4="; + rev = "6dc898658d554ce80e6fdd58904adea2fd0e30c8"; + hash = "sha256-7JMo2TU5JbPscrfI1qdz1P2bF6J8dTDhOqAQXxa3tm8="; }; }; @@ -29,15 +29,15 @@ rec { core = { x86_64-linux = { platform = "manylinux_2_34_x86_64"; - hash = "sha256-dFrlzhvvfJsDyStDHRdMpu54AaXioEfGSsIQTH5pUvs="; + hash = "sha256-2LlK9CZqV1KHrSzYRFc3Q6tP5Y9r+22SKTJ4B5N+reM="; }; aarch64-linux = { platform = "manylinux_2_34_aarch64"; - hash = "sha256-N24E9xOyRO7pXopRs+gSQM2nwHE214GfcntcoH7H7Kk="; + hash = "sha256-yIGjBbll5ZS4ixXCxkGbOY525DjsYeYJFsiy7/6SckA="; }; aarch64-darwin = { platform = "macosx_11_0_arm64"; - hash = "sha256-AsKxA5Wmy3NEQJ0kS6ylE33d0W86e9F494aiIkwyrcA="; + hash = "sha256-CeksnmwWNaFUnU4pey1KloTc01GyFu1/LrUc/FVHnEk="; }; }; diff --git a/pkgs/development/python-modules/semgrep/default.nix b/pkgs/development/python-modules/semgrep/default.nix index 69df6e0beeb5..853da1f60fca 100644 --- a/pkgs/development/python-modules/semgrep/default.nix +++ b/pkgs/development/python-modules/semgrep/default.nix @@ -5,8 +5,10 @@ semgrep-core, # check tools - git, + addBinToPathHook, + gitMinimal, pytestCheckHook, + writableTmpDirAsHomeHook, # python runtime dependencies attrs, @@ -15,6 +17,7 @@ click-option-group, colorama, defusedxml, + exceptiongroup, glom, jsonschema, mcp, @@ -51,33 +54,54 @@ let common = import ./common.nix { inherit lib; }; - semgrepBinPath = lib.makeBinPath [ semgrep-core ]; in -buildPythonPackage rec { - format = "setuptools"; +buildPythonPackage (finalAttrs: { pname = "semgrep"; inherit (common) version; + pyproject = true; + __structuredAttrs = true; + src = fetchFromGitHub { owner = "semgrep"; repo = "semgrep"; - rev = "v${version}"; + tag = "v${finalAttrs.version}"; hash = common.srcHash; }; + sourceRoot = "${finalAttrs.src.name}/cli"; + # prepare a subset of the submodules as we only need a handful # and there are many many submodules total postPatch = - (lib.concatStringsSep "\n" ( - lib.mapAttrsToList (path: submodule: '' - # substitute ${path} - # remove git submodule placeholder - rm -r ${path} - # link submodule - ln -s ${submodule}/ ${path} - '') passthru.submodulesSubset - )) + lib.concatStringsSep "\n" ( + lib.mapAttrsToList ( + path: submodule: + let + path' = lib.removePrefix "cli/" path; + in + '' + # substitute ${path'} + # remove git submodule placeholder + rm -r ${path'} + # link submodule + ln -s ${submodule}/ ${path'} + '' + ) finalAttrs.passthru.submodulesSubset + ) + # hardcode the path to the shared `semgrep-core` binary so that it does not + # need to be copied into the wheel nor be present on PATH at runtime. + # There are two independent lookups: one in the library and one in the CLI + # entrypoint (which execs semgrep-core directly). + '' - cd cli + substituteInPlace src/semgrep/semgrep_core.py \ + --replace-fail \ + 'ret = compute_executable_path("semgrep-core")' \ + 'ret = "${lib.getExe semgrep-core}"' + + substituteInPlace src/semgrep/console_scripts/entrypoint.py \ + --replace-fail \ + 'path = shutil.which(core)' \ + 'path = "${lib.getExe semgrep-core}" if not pro else shutil.which(core)' ''; # tell cli/setup.py to not copy semgrep-core into the result @@ -87,9 +111,16 @@ buildPythonPackage rec { pythonRelaxDeps = [ "boltons" + "click" + "exceptiongroup" "glom" + "jsonschema" + "mcp" + "opentelemetry-api" + "opentelemetry-exporter-otlp-proto-http" + "opentelemetry-sdk" + "wcmatch" ]; - dependencies = [ attrs boltons @@ -97,6 +128,7 @@ buildPythonPackage rec { click-option-group colorama defusedxml + exceptiongroup glom jsonschema mcp @@ -119,11 +151,11 @@ buildPythonPackage rec { wcmatch ]; - doCheck = true; - nativeCheckInputs = [ - git + addBinToPathHook + gitMinimal pytestCheckHook + writableTmpDirAsHomeHook flaky pytest-asyncio @@ -152,35 +184,18 @@ buildPythonPackage rec { "test_send" # many child tests require networking to download files "TestConfigLoaderForProducts" + # require networking (pro install metrics are sent to semgrep.dev) + "test_install_command_download_error_records_download_reason" + "test_install_command_metrics_off" + "test_install_command_sends_failure_metrics" + "test_install_command_sends_metrics_when_logged_in" + "test_run_install_success_records_outcome" + "test_run_install_version_check_failure_records_error" ]; - preCheck = '' - # tests need a home directory - export HOME="$(mktemp -d)" - - # tests need access to `semgrep-core` - export OLD_PATH="$PATH" - export PATH="$PATH:${semgrepBinPath}" - ''; - - postCheck = '' - export PATH="$OLD_PATH" - unset OLD_PATH - ''; - - # since we stop cli/setup.py from finding semgrep-core and copying it into - # the result we need to provide it on the PATH - preFixup = '' - makeWrapperArgs+=(--prefix PATH : ${semgrepBinPath}) - ''; - - postInstall = '' - chmod +x $out/bin/{,py}semgrep - ''; - passthru = { inherit common semgrep-core; - submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules; + submodulesSubset = lib.mapAttrs (_: args: fetchFromGitHub args) common.submodules; updateScript = ./update.sh; }; @@ -188,4 +203,4 @@ buildPythonPackage rec { description = common.meta.description + " - cli"; inherit (semgrep-core.meta) platforms; }; -} +}) diff --git a/pkgs/development/python-modules/semgrep/semgrep-core.nix b/pkgs/development/python-modules/semgrep/semgrep-core.nix index b71eecfdcbe4..65949ff664b4 100644 --- a/pkgs/development/python-modules/semgrep/semgrep-core.nix +++ b/pkgs/development/python-modules/semgrep/semgrep-core.nix @@ -3,17 +3,20 @@ stdenv, fetchPypi, - autoPatchelfHook, + patchelf, unzip, }: let common = import ./common.nix { inherit lib; }; in -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "semgrep-core"; inherit (common) version; + __structuredAttrs = true; + strictDeps = true; + # fetch pre-built semgrep-core since the ocaml build is complex and relies on # the opam package manager at some point # pulling it out of the python wheel as r2c no longer release a built binary @@ -22,22 +25,23 @@ stdenv.mkDerivation rec { let inherit (stdenv.hostPlatform) system; data = common.core.${system} or (throw "Unsupported system: ${system}"); + python = common.pythonWheelTag; in - fetchPypi rec { + fetchPypi { pname = "semgrep"; - inherit version; + inherit (finalAttrs) version; format = "wheel"; dist = python; - python = common.pythonWheelTag; + inherit python; inherit (data) platform hash; }; nativeBuildInputs = [ unzip ] - ++ lib.optional stdenv.hostPlatform.isLinux autoPatchelfHook; - - buildInputs = lib.optional stdenv.hostPlatform.isLinux stdenv.cc.cc.lib; + ++ lib.optionals stdenv.hostPlatform.isLinux [ + patchelf + ]; # _tryUnzip from unzip's setup-hook doesn't recognise .whl # "do not know how to unpack source archive" @@ -53,26 +57,36 @@ stdenv.mkDerivation rec { installPhase = '' runHook preInstall - install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core + install -Dm 755 -t $out/bin semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/semgrep-core # copy bundled libs as well # keeping them in bin/libs matches the layout in the wheel - if [ -d semgrep-${version}.data/purelib/semgrep/bin/libs ]; then + if [ -d semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/libs ]; then mkdir -p $out/bin/libs - cp -rf semgrep-${version}.data/purelib/semgrep/bin/libs/* $out/bin/libs/ - - # help autoPatchelfHook find these libs - if [ -n "''${autoPatchelfHook:-}" ]; then - appendAutoPatchelfSearchPath $out/bin/libs - fi + cp -rf semgrep-${finalAttrs.version}.data/purelib/semgrep/bin/libs/* $out/bin/libs/ fi runHook postInstall ''; + # Multiple rewrites of this large binary corrupt it on aarch64: patchelf leaves `.dynstr` + # outside any loadable segment, so ld.so segfaults reading the rpath at startup (patchelf bug, + # see https://github.com/NixOS/patchelf/issues/244). So we avoid autoPatchelfHook, disable + # stdenv's own `strip` and `patchelf --shrink-rpath`, and fix the interpreter and rpath in a + # single patchelf call below. The bundled bin/libs reference each other via $ORIGIN, so only the + # main binary needs it. + dontPatchELF = true; + dontStrip = true; + postFixup = lib.optionalString stdenv.hostPlatform.isLinux '' + patchelf \ + --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ + --set-rpath "$out/bin/libs:${lib.makeLibraryPath [ stdenv.cc.libc ]}" \ + $out/bin/semgrep-core + ''; + meta = common.meta // { description = common.meta.description + " - core binary"; mainProgram = "semgrep-core"; sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ]; platforms = lib.attrNames common.core; }; -} +})