diff --git a/pkgs/development/python-modules/s3torchconnectorclient/default.nix b/pkgs/development/python-modules/s3torchconnectorclient/default.nix new file mode 100644 index 000000000000..2db84b4f68ba --- /dev/null +++ b/pkgs/development/python-modules/s3torchconnectorclient/default.nix @@ -0,0 +1,88 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + boto3, + pytestCheckHook, + cmake, + hypothesis, + rustPackages, + setuptools-rust, + torch, +}: + +let + inherit (rustPackages) rustPlatform rustc cargo; + +in +buildPythonPackage rec { + pname = "s3torchconnectorclient"; + version = "1.5.0"; + format = "pyproject"; + __structuredAttrs = true; + + src = fetchFromGitHub { + owner = "awslabs"; + repo = "s3-connector-for-pytorch"; + rev = "v${version}"; + hash = "sha256-ovy/VUWTYMQ3wbmLptqj6l+uVwl4Gbkt3OpPUPayuLI="; + }; + + sourceRoot = "${src.name}/s3torchconnectorclient"; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit src sourceRoot; + hash = "sha256-xYfBnyZM43FNxPnnP6a45ZLCCve/B3713RNmgG0LNBc="; + }; + + nativeBuildInputs = [ + cargo + cmake + rustc + rustPlatform.cargoSetupHook + rustPlatform.bindgenHook + setuptools-rust + ]; + + # Patch metrics-0.24.1 to fix E0521 borrow-checker error under newer rustc. + # Backport of the fix from metrics 0.24.2. See: https://github.com/rust-lang/rust/issues/141402 + postPatch = '' + patch -d $cargoDepsCopy/*/metrics-0.24.1 -p1 < ${./fix-metrics-0.24.1-E0521.patch} + ''; + + # cmake is needed/used by the rust toolchain, we don't want Nix to run it directly + dontUseCmakeConfigure = true; + + env = { + CFLAGS = "-Wno-stringop-overflow -Wno-array-bounds -Wno-restrict"; + # pyo3-0.24.1 declares support only up to Python 3.13; use stable ABI for forward compat + PYO3_USE_ABI3_FORWARD_COMPATIBILITY = "1"; + }; + + dependencies = [ + boto3 + torch + ]; + + pythonImportsCheck = [ "s3torchconnectorclient" ]; + + nativeCheckInputs = [ + pytestCheckHook + hypothesis + ]; + + disabledTestPaths = [ + # integration tests access S3 + "python/tst/integration/" + # unit tests for S3 client creation require AWS credentials + TLS trust store + "python/tst/unit/test_mountpoint_s3_client.py" + ]; + + meta = with lib; { + description = "Low-level S3 client for PyTorch data loading"; + homepage = "https://github.com/awslabs/s3-connector-for-pytorch"; + changelog = "https://github.com/awslabs/s3-connector-for-pytorch/releases/tag/v${version}"; + license = licenses.bsd3; + maintainers = with maintainers; [ jherland ]; + }; +} diff --git a/pkgs/development/python-modules/s3torchconnectorclient/fix-metrics-0.24.1-E0521.patch b/pkgs/development/python-modules/s3torchconnectorclient/fix-metrics-0.24.1-E0521.patch new file mode 100644 index 000000000000..390c1732b8e9 --- /dev/null +++ b/pkgs/development/python-modules/s3torchconnectorclient/fix-metrics-0.24.1-E0521.patch @@ -0,0 +1,27 @@ +--- a/src/recorder/mod.rs ++++ b/src/recorder/mod.rs +@@ -139,11 +139,19 @@ + + impl<'a> LocalRecorderGuard<'a> { + /// Creates a new `LocalRecorderGuard` and sets the thread-local recorder. +- fn new(recorder: &'a dyn Recorder) -> Self { +- // SAFETY: While we take a lifetime-less pointer to the given reference, the reference we derive _from_ the +- // pointer is given the same lifetime of the reference used to construct the guard -- captured in the guard type +- // itself -- and so derived references never outlive the source reference. +- let recorder_ptr = unsafe { NonNull::new_unchecked(recorder as *const _ as *mut _) }; ++ fn new(recorder: &'a (dyn Recorder + 'a)) -> Self { ++ // SAFETY: We extend `'a` to `'static` to satisfy the signature of `LOCAL_RECORDER`, which ++ // has an implied `'static` bound on `dyn Recorder`. We enforce that all usages of `LOCAL_RECORDER` ++ // are limited to `'a` as we mediate its access entirely through `LocalRecorderGuard<'a>`. ++ let recorder_ptr = unsafe { ++ std::mem::transmute::<*const (dyn Recorder + 'a), *mut (dyn Recorder + 'static)>( ++ recorder as &'a (dyn Recorder + 'a), ++ ) ++ }; ++ // SAFETY: While we take a lifetime-less pointer to the given reference, the reference we derive _from_ the ++ // pointer is given the same lifetime of the reference used to construct the guard -- captured in the guard type ++ // itself -- and so derived references never outlive the source reference. ++ let recorder_ptr = unsafe { NonNull::new_unchecked(recorder_ptr) }; + + let prev_recorder = + LOCAL_RECORDER.with(|local_recorder| local_recorder.replace(Some(recorder_ptr))); diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 38c11acee55d..33447fd4ffaa 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18455,6 +18455,8 @@ self: super: with self; { s3fs = callPackage ../development/python-modules/s3fs { }; + s3torchconnectorclient = callPackage ../development/python-modules/s3torchconnectorclient { }; + s3transfer = callPackage ../development/python-modules/s3transfer { }; sabctools = callPackage ../development/python-modules/sabctools { };