diff --git a/pkgs/by-name/ow/owocr/package.nix b/pkgs/by-name/ow/owocr/package.nix index 999f542a8a82..fed2260b6573 100644 --- a/pkgs/by-name/ow/owocr/package.nix +++ b/pkgs/by-name/ow/owocr/package.nix @@ -7,6 +7,7 @@ python3Packages.buildPythonApplication (finalAttrs: { pname = "owocr"; version = "1.26.8"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "AuroraWright"; @@ -62,7 +63,9 @@ python3Packages.buildPythonApplication (finalAttrs: { meta = { description = "Optical character recognition for Japanese text"; homepage = "https://github.com/AuroraWright/owocr"; + changelog = "https://github.com/AuroraWright/owocr/releases/tag/${finalAttrs.src.tag}"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ sigmanificient ]; + mainProgram = "owocr"; }; }) diff --git a/pkgs/development/python-modules/azure-ai-vision-imageanalysis/default.nix b/pkgs/development/python-modules/azure-ai-vision-imageanalysis/default.nix index 6d01ee1c78e4..00156c47e701 100644 --- a/pkgs/development/python-modules/azure-ai-vision-imageanalysis/default.nix +++ b/pkgs/development/python-modules/azure-ai-vision-imageanalysis/default.nix @@ -10,14 +10,15 @@ buildPythonPackage (finalAttrs: { pname = "azure-ai-vision-imageanalysis"; - version = "39.0.0"; + version = "1.0.0"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "Azure"; repo = "azure-sdk-for-python"; - tag = "azure-mgmt-containerservice_${finalAttrs.version}"; - hash = "sha256-zufXc8LR4STHi/jjV0bcLsifcHIif2m+3Q/KZlsSkRw="; + tag = "azure-ai-vision-imageanalysis_${finalAttrs.version}"; + hash = "sha256-gkOKD7koHmsApnylnlQ+/PkbhyUKwigeawznPlU1jvM="; }; sourceRoot = "${finalAttrs.src.name}/sdk/vision/azure-ai-vision-imageanalysis"; @@ -37,6 +38,7 @@ buildPythonPackage (finalAttrs: { meta = { description = "Azure Image Analysis client library for Python"; homepage = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/vision/azure-ai-vision-imageanalysis"; + changelog = "https://github.com/Azure/azure-sdk-for-python/blob/${finalAttrs.src.tag}/sdk/vision/azure-ai-vision-imageanalysis/CHANGELOG.md"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ sigmanificient ]; }; diff --git a/pkgs/development/python-modules/deepsearch-toolkit/default.nix b/pkgs/development/python-modules/deepsearch-toolkit/default.nix index 451c878b4480..6bcb4fb230e4 100644 --- a/pkgs/development/python-modules/deepsearch-toolkit/default.nix +++ b/pkgs/development/python-modules/deepsearch-toolkit/default.nix @@ -2,7 +2,11 @@ lib, buildPythonPackage, fetchFromGitHub, + + # build-system poetry-core, + + # dependencies certifi, docling-core, platformdirs, @@ -20,18 +24,18 @@ anyio, fastapi, uvicorn, - pytestCheckHook, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "deepsearch-toolkit"; version = "2.0.1"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "DS4SD"; repo = "deepsearch-toolkit"; - tag = "v${version}"; + tag = "v${finalAttrs.version}"; hash = "sha256-nrz9pvyA5gPIaKt6CsJOB9cLy3sXiWW5e1Rk4vtNIY8="; }; @@ -61,8 +65,7 @@ buildPythonPackage rec { "urllib3" ]; - optional-dependencies = rec { - all = api; + optional-dependencies = { api = [ anyio fastapi @@ -70,25 +73,19 @@ buildPythonPackage rec { ]; }; - pythonImportsCheck = [ - "deepsearch" - ]; + pythonImportsCheck = [ "deepsearch" ]; - nativeCheckInputs = [ - pytestCheckHook - ]; - - disabledTests = [ - # Tests require the creation of a deepsearch profile - "test_project_listing" - "test_system_info" - ]; + # The remaining tests either require the creation of a deepsearch profile, or + # exercise the deprecated `export_to_markdown`, which relies on the legacy + # document models docling-core removed in favor of empty import shims: + # https://github.com/docling-project/docling-core/pull/644 + doCheck = false; meta = { - changelog = "https://github.com/DS4SD/deepsearch-toolkit/blob/${src.tag}/CHANGELOG.md"; + changelog = "https://github.com/DS4SD/deepsearch-toolkit/blob/${finalAttrs.src.tag}/CHANGELOG.md"; description = "Interact with the Deep Search platform for new knowledge explorations and discoveries"; homepage = "https://github.com/DS4SD/deepsearch-toolkit"; license = lib.licenses.mit; maintainers = [ ]; }; -} +}) diff --git a/pkgs/development/python-modules/doclang/default.nix b/pkgs/development/python-modules/doclang/default.nix new file mode 100644 index 000000000000..2d4d290d5794 --- /dev/null +++ b/pkgs/development/python-modules/doclang/default.nix @@ -0,0 +1,67 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + # build-system + setuptools, + + # dependencies + lxml, + typer, + + # optional-dependencies + saxonche, + + # tests + pytestCheckHook, + python-docx, + versionCheckHook, +}: + +buildPythonPackage (finalAttrs: { + pname = "doclang"; + version = "0.7.3"; + pyproject = true; + __structuredAttrs = true; + + src = fetchFromGitHub { + owner = "doclang-project"; + repo = "doclang"; + tag = "v${finalAttrs.version}"; + hash = "sha256-U5QuYCTB2OTugfmvskMnoA71u1DUEVMZTi9H7C7PYEY="; + }; + + build-system = [ + setuptools + ]; + + dependencies = [ + lxml + typer + ]; + + optional-dependencies = { + schematron-saxon = [ + saxonche + ]; + }; + + pythonImportsCheck = [ "doclang" ]; + + nativeCheckInputs = [ + pytestCheckHook + python-docx + saxonche + versionCheckHook + ]; + + meta = { + description = "DocLang spec and reference toolkit"; + homepage = "https://github.com/doclang-project/doclang"; + changelog = "https://github.com/doclang-project/doclang/blob/${finalAttrs.src.rev}/CHANGELOG.md"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ GaetanLepage ]; + mainProgram = "doclang"; + }; +}) diff --git a/pkgs/development/python-modules/docling-core/default.nix b/pkgs/development/python-modules/docling-core/default.nix index a5386e11148b..483088d3c772 100644 --- a/pkgs/development/python-modules/docling-core/default.nix +++ b/pkgs/development/python-modules/docling-core/default.nix @@ -9,12 +9,14 @@ # dependencies defusedxml, + doclang, jsonref, jsonschema, latex2mathml, pandas, pillow, pydantic, + pydantic-settings, pyyaml, semchunk, tabulate, @@ -26,20 +28,24 @@ # tests gitpython, jsondiff, + opencv-python-headless, pytestCheckHook, requests, + saxonche, + universal-pathlib, }: buildPythonPackage (finalAttrs: { pname = "docling-core"; - version = "2.73.0"; + version = "2.90.0"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "docling-project"; repo = "docling-core"; tag = "v${finalAttrs.version}"; - hash = "sha256-aK+XHZjKsmFPgRv0oDed1CdBwZags/zcALumgcfQjjY="; + hash = "sha256-giPdZKVwR+Gvo4hpn/hUhACMuZwCEqhZwqph+wOrqgk="; }; build-system = [ @@ -50,16 +56,19 @@ buildPythonPackage (finalAttrs: { pythonRelaxDeps = [ "defusedxml" "pillow" + "pydantic-settings" "typer" ]; dependencies = [ defusedxml + doclang jsonref jsonschema latex2mathml pandas pillow pydantic + pydantic-settings pyyaml semchunk tabulate @@ -74,22 +83,31 @@ buildPythonPackage (finalAttrs: { nativeCheckInputs = [ gitpython jsondiff + # `ImageRef.from_pil` encodes PNGs with OpenCV when available, and the test + # reference files are generated with it: https://github.com/docling-project/docling-core/pull/562 + opencv-python-headless pytestCheckHook requests + saxonche + universal-pathlib ]; disabledTestPaths = [ # attempts to download models + "test/test_chunk_expander.py" "test/test_code_chunker.py" "test/test_code_chunking_strategy.py" "test/test_hybrid_chunker.py" "test/test_line_chunker.py" + + # Requires unpackaged dclq + "packages/dclq/tests/test_cli.py" ]; meta = { - changelog = "https://github.com/docling-project/docling-core/blob/${finalAttrs.src.tag}/CHANGELOG.md"; description = "Python library to define and validate data types in Docling"; homepage = "https://github.com/docling-project/docling-core"; + changelog = "https://github.com/docling-project/docling-core/blob/${finalAttrs.src.tag}/CHANGELOG.md"; license = lib.licenses.mit; maintainers = [ ]; }; diff --git a/pkgs/development/python-modules/docling-ibm-models/default.nix b/pkgs/development/python-modules/docling-ibm-models/default.nix index c1f5e064327d..062f49b60202 100644 --- a/pkgs/development/python-modules/docling-ibm-models/default.nix +++ b/pkgs/development/python-modules/docling-ibm-models/default.nix @@ -30,14 +30,15 @@ buildPythonPackage (finalAttrs: { pname = "docling-ibm-models"; - version = "3.13.2"; + version = "3.13.3"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "docling-project"; repo = "docling-ibm-models"; tag = "v${finalAttrs.version}"; - hash = "sha256-PYIcsKffrsZl4f9SFinUdrvUVEXm1fO/n7ZxXuiiByU="; + hash = "sha256-b//8cPXEPGNYXVd6wAQcNDh240i3gjdRPFzz4tYfeUk="; }; build-system = [ diff --git a/pkgs/development/python-modules/docling-jobkit/default.nix b/pkgs/development/python-modules/docling-jobkit/default.nix index c4157254da97..e6ddff51f826 100644 --- a/pkgs/development/python-modules/docling-jobkit/default.nix +++ b/pkgs/development/python-modules/docling-jobkit/default.nix @@ -3,92 +3,131 @@ stdenv, buildPythonPackage, fetchFromGitHub, + pythonAtLeast, + pythonOlder, # build-system hatchling, - poetry-core, # dependencies - docling, - pydantic-settings, - typer, boto3, - pandas, - fastparquet, - pyarrow, + docling-slim, httpx, + pandas, + pydantic-settings, - # optional dependencies - ray, + # optional-dependencies + # rq: rq, msgpack, + # ray: + redis, + psutil, + ray, + # gdrive: + google-api-python-client, + google-auth-oauthlib, + # gcloudstorage: + google-cloud-storage, + # azure: + azure-storage-blob, + # opensearch: + opensearch-py, # tests - pytestCheckHook, + aiohttp, pytest-asyncio, + pytestCheckHook, + starlette, + redisTestHook, writableTmpDirAsHomeHook, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "docling-jobkit"; - version = "1.8.1"; + version = "3.2.0"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "docling-project"; repo = "docling-jobkit"; - tag = "v${version}"; - hash = "sha256-9DzQY/XMmx/8XP1bMYZYl+Bp7AVcYfuv3MtO6lvQ/24="; + tag = "v${finalAttrs.version}"; + hash = "sha256-4OUNDn90jGWqHcs9QPpOKqYnKDxH+N/YUeszY21nhMo="; }; build-system = [ hatchling - poetry-core ]; - dependencies = [ - docling - pydantic-settings - typer - boto3 - pandas - fastparquet - pyarrow - httpx + pythonRelaxDeps = [ + "pandas" ]; + dependencies = [ + boto3 + docling-slim + httpx + pandas + pydantic-settings + ] + ++ docling-slim.optional-dependencies.standard; optional-dependencies = { - ray = [ ray ]; + vlm = docling-slim.optional-dependencies.models-vlm-inline; rq = [ rq msgpack ]; + ray = [ + msgpack + psutil + ray + ] + ++ ray.optional-dependencies.serve + ++ redis.optional-dependencies.hiredis + ++ lib.optionals ((pythonAtLeast "3.11") && (pythonOlder "3.14")) [ + # cloudflare-sdk + ]; + gdrive = [ + google-api-python-client + google-auth-oauthlib + ]; + gcloudstorage = [ + google-cloud-storage + ]; + azure = [ + azure-storage-blob + ]; + opensearch = [ + opensearch-py + ]; }; - pythonRelaxDeps = [ - "boto3" - "pandas" - "pyarrow" - ]; - pythonImportsCheck = [ "docling" "docling_jobkit" ]; nativeCheckInputs = [ - pytestCheckHook + aiohttp pytest-asyncio + pytestCheckHook + ray + redisTestHook + starlette writableTmpDirAsHomeHook ] - ++ optional-dependencies.rq; + ++ lib.concatAttrValues finalAttrs.passthru.optional-dependencies; disabledTests = [ # requires network access "test_chunk_file" + "test_chunker_manager_shared_across_workers" + "test_clear_converters_clears_caches" "test_convert_file" - "test_convert_warmup" "test_convert_url" + "test_convert_warmup" + "test_convert_with_callbacks" "test_replicated_convert" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ @@ -97,11 +136,13 @@ buildPythonPackage rec { "test_options_validator" ]; + __darwinAllowLocalNetworking = true; + meta = { - changelog = "https://github.com/docling-project/docling-jobkit/blob/${src.tag}/CHANGELOG.md"; description = "Running a distributed job processing documents with Docling"; homepage = "https://github.com/docling-project/docling-jobkit"; + changelog = "https://github.com/docling-project/docling-jobkit/blob/${finalAttrs.src.tag}/CHANGELOG.md"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ codgician ]; }; -} +}) diff --git a/pkgs/development/python-modules/docling-parse/default.nix b/pkgs/development/python-modules/docling-parse/default.nix index 0cfb2b047488..549b155ff045 100644 --- a/pkgs/development/python-modules/docling-parse/default.nix +++ b/pkgs/development/python-modules/docling-parse/default.nix @@ -2,108 +2,110 @@ lib, buildPythonPackage, fetchFromGitHub, + + # build-system cmake, - pkg-config, - cxxopts, - setuptools, pybind11, - zlib, - nlohmann_json, - utf8cpp, + setuptools, + + # nativeBuildInputs + pkg-config, + + # buildInputs + blend2d, + cxxopts, + freetype, + lcms2, libjpeg, - qpdf, loguru-cpp, - # python dependencies - tabulate, + nlohmann_json, + openjpeg, + qpdf, + utf8cpp, + zlib, + + # dependencies + docling-core, pillow, pydantic, - docling-core, - pytestCheckHook, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "docling-parse"; - version = "5.0.0"; + version = "7.8.1"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "docling-project"; repo = "docling-parse"; - tag = "v${version}"; - hash = "sha256-qxD3ryU1jXf8Gm5/IiG2NTOnRgA6HADPfgBj6Kn+Pj4="; + tag = "v${finalAttrs.version}"; + hash = "sha256-WTXWIJWwxfBdPnoCvo1sq7o9itG/MUX3FNz1GGtvaVo="; }; + # cibuildwheel and delocate are listed as build-system dependencies but are only used in CI to + # build wheels. postPatch = '' substituteInPlace pyproject.toml \ --replace-fail \ - '"cmake>=3.27.0,<4.0.0"' \ - '"cmake>=3.27.0"' + "cmake>=3.27.0,<4.0.0" \ + "cmake>=3.27.0" \ + --replace-fail \ + "\"cibuildwheel>=2.19.2,<4.0.0; python_version >= '3.11'\"," \ + "" \ + --replace-fail \ + '"delocate>=0.11.0,<1.0.0",' \ + "" ''; dontUseCmakeConfigure = true; nativeBuildInputs = [ - cmake pkg-config ]; build-system = [ + cmake + pybind11 setuptools ]; - env.NIX_CFLAGS_COMPILE = "-I${lib.getDev utf8cpp}/include/utf8cpp"; + env = { + NIX_CFLAGS_COMPILE = "-I${lib.getDev utf8cpp}/include/utf8cpp"; + USE_SYSTEM_DEPS = "ON"; + }; buildInputs = [ - pybind11 + blend2d cxxopts + freetype + lcms2 libjpeg loguru-cpp nlohmann_json + openjpeg qpdf utf8cpp zlib ]; - env.USE_SYSTEM_DEPS = true; - - cmakeFlags = [ - "-DUSE_SYSTEM_DEPS=True" - ]; - dependencies = [ - tabulate + docling-core pillow pydantic - docling-core ]; - pythonRelaxDeps = [ - "pydantic" - "pillow" - ]; + pythonImportsCheck = [ "docling_parse" ]; - # Listed as runtime dependencies but only used in CI to build wheels - preBuild = '' - sed -i '/cibuildwheel/d' pyproject.toml - sed -i '/delocate/d' pyproject.toml - ''; - - pythonImportsCheck = [ - "docling_parse" - ]; - - nativeCheckInputs = [ - pytestCheckHook - ]; + # The whole test suite requires the test corpus, which `tests/conftest.py` unconditionally + # downloads from HuggingFace in `pytest_sessionstart`. + doCheck = false; meta = { - changelog = "https://github.com/DS4SD/docling-parse/blob/${src.tag}/CHANGELOG.md"; + changelog = "https://github.com/DS4SD/docling-parse/blob/${finalAttrs.src.tag}/CHANGELOG.md"; description = "Simple package to extract text with coordinates from programmatic PDFs"; homepage = "https://github.com/DS4SD/docling-parse"; license = lib.licenses.mit; maintainers = [ ]; - # error: no matching conversion for functional-style cast from 'bool' to 'nlohmann::basic_json<>' - # See https://github.com/docling-project/docling-parse/issues/172 for context - broken = true; }; -} +}) diff --git a/pkgs/development/python-modules/docling-serve/default.nix b/pkgs/development/python-modules/docling-serve/default.nix index 199d12993e35..0048ace5cd1e 100644 --- a/pkgs/development/python-modules/docling-serve/default.nix +++ b/pkgs/development/python-modules/docling-serve/default.nix @@ -2,65 +2,88 @@ lib, buildPythonPackage, fetchFromGitHub, + + # build-system hatchling, - setuptools-scm, - # python dependencies - docling, + + # dependencies + docling-core, docling-jobkit, docling-mcp, + docling-slim, fastapi, httpx, + opentelemetry-api, + opentelemetry-exporter-otlp, + opentelemetry-exporter-prometheus, + opentelemetry-instrumentation-fastapi, + opentelemetry-sdk, + prometheus-client, + pydantic, pydantic-settings, python-multipart, scalar-fastapi, + typer, uvicorn, websockets, + + # optional-dependencies + # ui: + gradio, + # tesserocr: tesserocr, - typer, + # easyocr: + easyocr, + # rapidocr: rapidocr, onnxruntime, - torch, - torchvision, - gradio, - nodejs, - which, + # flash-attn: + flash-attn, + + # tests + versionCheckHook, + withUI ? false, withTesserocr ? false, withRapidocr ? false, withCPU ? false, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "docling-serve"; - version = "1.10.0"; + version = "1.29.0"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "docling-project"; repo = "docling-serve"; - tag = "v${version}"; - hash = "sha256-g0ATehTRtrqgTjvMTs+yvFdFwXTZ8AWsO+Hljwlcbto="; + tag = "v${finalAttrs.version}"; + hash = "sha256-sYpi7vgeUH2xw2Jq8Rgou8SsS8aq+TWgq29gqv/xNnU="; }; build-system = [ hatchling - setuptools-scm ]; pythonRelaxDeps = [ - "websockets" + "fastapi" + "opentelemetry-instrumentation-fastapi" ]; - - pythonRemoveDeps = [ - "mlx-vlm" # not yet available on nixpkgs - ]; - dependencies = [ - docling + docling-core docling-jobkit docling-mcp + docling-slim fastapi httpx + opentelemetry-api + opentelemetry-exporter-otlp + opentelemetry-exporter-prometheus + opentelemetry-instrumentation-fastapi + opentelemetry-sdk + prometheus-client + pydantic pydantic-settings python-multipart scalar-fastapi @@ -68,27 +91,38 @@ buildPythonPackage rec { uvicorn websockets ] - ++ lib.optionals withUI optional-dependencies.ui - ++ lib.optionals withTesserocr optional-dependencies.tesserocr - ++ lib.optionals withRapidocr optional-dependencies.rapidocr - ++ lib.optionals withCPU optional-dependencies.cpu; + ++ docling-slim.optional-dependencies.standard + ++ docling-slim.optional-dependencies.service-client + ++ docling-slim.optional-dependencies.models-remote + ++ docling-slim.optional-dependencies.format-opendocument + ++ docling-jobkit.optional-dependencies.rq + ++ docling-jobkit.optional-dependencies.ray + ++ docling-jobkit.optional-dependencies.azure + ++ docling-jobkit.optional-dependencies.gcloudstorage + ++ docling-jobkit.optional-dependencies.gdrive + ++ fastapi.optional-dependencies.standard + ++ uvicorn.optional-dependencies.standard + ++ lib.optionals withUI finalAttrs.passthru.optional-dependencies.ui + ++ lib.optionals withTesserocr finalAttrs.passthru.optional-dependencies.tesserocr + ++ lib.optionals withRapidocr finalAttrs.passthru.optional-dependencies.rapidocr + ++ lib.optionals withCPU finalAttrs.passthru.optional-dependencies.cpu; optional-dependencies = { ui = [ gradio - nodejs - which ]; tesserocr = [ tesserocr ]; + easyocr = [ + easyocr + ]; rapidocr = [ rapidocr onnxruntime ]; - cpu = [ - torch - torchvision + flash-attn = [ + flash-attn ]; }; @@ -96,15 +130,17 @@ buildPythonPackage rec { "docling_serve" ]; - # Require network - doCheck = false; + # Python tests require network + nativeCheckInputs = [ + versionCheckHook + ]; meta = { - changelog = "https://github.com/docling-project/docling-serve/blob/${src.tag}/CHANGELOG.md"; description = "Running Docling as an API service"; homepage = "https://github.com/docling-project/docling-serve"; + changelog = "https://github.com/docling-project/docling-serve/blob/${finalAttrs.src.tag}/CHANGELOG.md"; license = lib.licenses.mit; mainProgram = "docling-serve"; maintainers = [ ]; }; -} +}) diff --git a/pkgs/development/python-modules/docling-slim/default.nix b/pkgs/development/python-modules/docling-slim/default.nix new file mode 100644 index 000000000000..368a0e9cf5a5 --- /dev/null +++ b/pkgs/development/python-modules/docling-slim/default.nix @@ -0,0 +1,368 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + # build-system + hatchling, + + # dependencies + certifi, + docling-core, + filetype, + pluggy, + pydantic, + pydantic-settings, + requests, + tqdm, + + # optional-dependencies + # cli: + python-dotenv, + rich, + typer, + # convert-core: + numpy, + pillow, + rtree, + scipy, + # extract-core: + polyfactory, + # feat-ocr-easyocr: + easyocr, + scikit-image, + # feat-ocr-rapidocr-onnx: + onnxruntime, + rapidocr, + # feat-ocr-tesserocr: + pandas, + tesserocr, + # format-audio: + numba, + openai-whisper, + # format-docx: + python-docx, + # format-email: + mail-parser, + # format-html: + beautifulsoup4, + # format-html-render: + playwright, + # format-latex: + pylatexenc, + # format-markdown: + marko, + # format-opendocument: + odfdo, + # format-pdf-docling: + docling-parse, + pypdfium2, + # format-pptx: + python-pptx, + # format-video: + librosa, + scikit-learn, + soundfile, + # format-xlsx: + openpyxl, + #  format-xml-jats: + lxml, + # format-xml-uspto: + defusedxml, + # models-local: + accelerate, + docling-ibm-models, + huggingface-hub, + torch, + torchvision, + # models-remote: + tritonclient, + # models-vlm-inline: + peft, + transformers, + # service-client: + httpx, + websockets, + + # tests + ffmpeg-headless, + gitMinimal, + pytestCheckHook, + writableTmpDirAsHomeHook, +}: + +buildPythonPackage (finalAttrs: { + pname = "docling-slim"; + version = "2.118.0"; + pyproject = true; + __structuredAttrs = true; + + src = fetchFromGitHub { + owner = "docling-project"; + repo = "docling"; + tag = "v${finalAttrs.version}"; + hash = "sha256-99XxwibuvorgXXvqZtHHjJ4wvjoQdTNC36S6GnQTMXQ="; + }; + + build-system = [ + hatchling + ]; + + dependencies = [ + certifi + docling-core + filetype + pluggy + pydantic + pydantic-settings + requests + tqdm + ]; + + optional-dependencies = rec { + cli = [ + python-dotenv + rich + typer + ]; + convert-core = [ + numpy + pillow + rtree + scipy + ]; + extract-core = convert-core ++ [ + polyfactory + ]; + # docling-core[chunking], whose extra deps nixpkgs already ships unconditionally + feat-chunking = [ + docling-core + ]; + feat-ocr-easyocr = [ + easyocr + scikit-image + ]; + # ocrmac is darwin-only and not packaged in nixpkgs + feat-ocr-mac = [ ]; + feat-ocr-rapidocr = [ + rapidocr + ]; + feat-ocr-rapidocr-onnx = [ + onnxruntime + rapidocr + ]; + feat-ocr-tesserocr = [ + pandas + tesserocr + ]; + # whisper-s2t-reborn is not packaged in nixpkgs + format-audio = [ + numba + openai-whisper + ]; + format-docx = [ + python-docx + ]; + format-email = format-html ++ [ + mail-parser + ]; + format-html = [ + beautifulsoup4 + ]; + format-html-render = [ + playwright + ]; + format-latex = [ + pylatexenc + ]; + format-markdown = [ + marko + ]; + format-office = format-docx ++ format-pptx ++ format-xlsx; + format-opendocument = [ + odfdo + ]; + format-pdf = format-pdf-pypdfium2 ++ format-pdf-docling; + format-pdf-docling = [ + docling-parse + pypdfium2 + ]; + format-pdf-pypdfium2 = [ + pypdfium2 + ]; + format-pptx = [ + python-pptx + ]; + # resemblyzer is not packaged in nixpkgs + format-video = format-audio ++ [ + librosa + scikit-learn + soundfile + ]; + format-web = format-html ++ format-markdown; + format-xlsx = [ + openpyxl + ]; + format-xml-jats = format-html ++ [ + lxml + ]; + format-xml-uspto = format-html ++ [ + defusedxml + ]; + # arelle-release is not packaged in nixpkgs + format-xml-xbrl = [ ]; + models-local = [ + accelerate + defusedxml + docling-ibm-models + huggingface-hub + torch + torchvision + ]; + models-onnxruntime = [ + onnxruntime + # onnxruntime-gpu + ]; + models-remote = [ + tritonclient + ]; + # qwen-vl-utils is not packaged in nixpkgs + models-vlm-inline = [ + accelerate + peft + transformers + ]; + service-client = [ + httpx + python-dotenv + rich + typer + websockets + ]; + + standard = + cli + ++ extract-core + ++ feat-chunking + ++ feat-ocr-rapidocr + ++ format-email + ++ format-latex + ++ format-office + ++ format-pdf + ++ format-web + ++ models-local + ++ service-client; + + # format-video is deliberately excluded, as upstream does + all = + standard + ++ feat-ocr-easyocr + ++ feat-ocr-mac + ++ feat-ocr-tesserocr + ++ format-audio + ++ format-html-render + ++ format-xml-jats + ++ format-xml-uspto + ++ format-xml-xbrl + ++ models-onnxruntime + ++ models-remote + ++ models-vlm-inline; + }; + + pythonImportsCheck = [ "docling" ]; + + nativeCheckInputs = [ + beautifulsoup4 + docling-ibm-models + docling-parse + easyocr + ffmpeg-headless + gitMinimal + mail-parser + marko + openpyxl + pylatexenc + pypdfium2 + pytestCheckHook + python-docx + python-pptx + rapidocr + rtree + scipy + websockets + writableTmpDirAsHomeHook + ]; + + disabledTests = [ + # Downloads models from HuggingFace or fetches remote documents + "test_cli_convert" + "test_cli_html_fetches_remote_images_with_separate_headers" + "test_confidence" + "test_convert_path" + "test_convert_remote_too_large_filesize_limit_wout_exception" + "test_convert_stream" + "test_document_timeout" + "test_e2e_conversions" + "test_e2e_html_conversion_with_images" + "test_extraction_with_dict_template" + "test_extraction_with_pydantic_class_template" + "test_extraction_with_pydantic_instance_template" + "test_extraction_with_string_template" + "test_fetch_remote_images" + "test_fetch_remote_images_with_custom_headers" + "test_get_text_from_rect_rotated" + "test_load_image_data_enforces_size_limit" + "test_page_error_carries_page_no" + "test_page_range" + "test_parser_backends" + "test_pipeline_cache_after_initialize" + "test_pipeline_cache_with_chart_extraction" + "test_threaded_and_standard_backends_convert_with_standard_pipeline" + "test_threaded_pipeline_multiple_documents" + "test_threaded_pipeline_page_range" + "test_threaded_pipeline_with_pypdfium_backend" + + # Require openai-whisper, which is too heavy a dependency to pull in just for these + "test_asr_pipeline_conversion" + "test_asr_pipeline_with_silent_audio" + "test_has_text_and_determine_status_helpers" + "test_native_and_mlx_transcribe_language_handling" + "test_native_distil_artifacts_path_missing_checkpoint_raises" + "test_native_init_with_artifacts_path_and_device_logging" + "test_native_run_failure_sets_status" + "test_native_run_success_with_bytesio_builds_document" + + # The sandbox pins OMP_NUM_THREADS=1, while the test expects the built-in default of 4 + "test_accelerator_options" + + # Needs LibreOffice on PATH to render DrawingML, and a Pillow with WMF support + "test_e2e_docx_conversions" + ]; + + disabledTestPaths = [ + # Download models from HuggingFace or fetch remote documents + "tests/test_backend_webp.py" + "tests/test_code_formula.py" + "tests/test_conversion_result_json.py" + "tests/test_document_picture_classifier.py" + "tests/test_e2e_conversion.py" + "tests/test_failed_pages.py" + "tests/test_heading_hierarchy_pdf.py" + "tests/test_layout_picture_table_overlap.py" + "tests/test_pdf_password.py" + + # arelle-release is not packaged in nixpkgs + "tests/test_backend_xbrl.py" + + # tesserocr is packaged without the tessdata it needs to initialize + "tests/test_e2e_ocr_conversion.py" + ]; + + meta = { + description = "Modular version of the Docling package"; + homepage = "https://github.com/docling-project/docling"; + changelog = "https://github.com/docling-project/docling/blob/${finalAttrs.src.rev}/CHANGELOG.md"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +}) diff --git a/pkgs/development/python-modules/docling/default.nix b/pkgs/development/python-modules/docling/default.nix index d58e3a9198a0..e53352e7e101 100644 --- a/pkgs/development/python-modules/docling/default.nix +++ b/pkgs/development/python-modules/docling/default.nix @@ -1,187 +1,47 @@ { lib, buildPythonPackage, - fetchFromGitHub, + docling-slim, # build system - poetry-core, - - # dependencies - accelerate, - beautifulsoup4, - certifi, - docling-core, - docling-ibm-models, - docling-parse, - easyocr, - filetype, - huggingface-hub, - lxml, - marko, - # ocrmac # not yet packaged - onnxruntime, - openpyxl, - pandas, - pillow, - pluggy, - pydantic, - pydantic-settings, - pylatexenc, - pypdfium2, - python-docx, - python-pptx, - rapidocr, - requests, - rtree, - scipy, - tesserocr, - tqdm, - transformers, - typer, - - # optional dependencies - # mkdocs-click # not yet packaged - mkdocs-jupyter, - mkdocs-material, - mkdocstrings, + hatchling, # tests - pytestCheckHook, - writableTmpDirAsHomeHook, + versionCheckHook, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "docling"; - version = "2.69.1"; + inherit (docling-slim) version src; pyproject = true; + __structuredAttrs = true; - src = fetchFromGitHub { - owner = "docling-project"; - repo = "docling"; - tag = "v${version}"; - hash = "sha256-r7jAah/tqLylPyyzrK0NW2ok66NVdb/V/YLV95McGC4="; - }; + sourceRoot = "${finalAttrs.src.name}/packages/docling"; build-system = [ - poetry-core + hatchling ]; + # Meta-package: it ships no Python module of its own, only the console scripts. `docling` itself + # is provided by docling-slim, which upstream pulls in with the `standard` extra. dependencies = [ - accelerate - beautifulsoup4 - certifi - docling-core - docling-ibm-models - docling-parse - easyocr - filetype - huggingface-hub - lxml - marko - # ocrmac # not yet packaged - onnxruntime - openpyxl - pandas - pillow - pluggy - pydantic - pydantic-settings - pylatexenc - pypdfium2 - python-docx - python-pptx - rapidocr - requests - rtree - scipy - tesserocr - tqdm - transformers - typer - ]; + docling-slim + ] + ++ docling-slim.optional-dependencies.standard; - pythonRelaxDeps = [ - "lxml" - "pypdfium2" - "pillow" - ]; - - optional-dependencies = { - ocrmac = [ - # ocrmac # not yet packaged - ]; - rapidocr = [ - onnxruntime - rapidocr - ]; - tesserocr = [ - tesserocr - ]; - - docs = [ - # mkdocs-click # not yet packaged - mkdocs-jupyter - mkdocs-material - mkdocstrings - # griffle-pydantic - ]; - }; + pythonImportsCheck = [ "docling" ]; + # No python tests in this subpackage, they all live in docling-slim nativeCheckInputs = [ - pytestCheckHook - writableTmpDirAsHomeHook - ]; - - pythonImportsCheck = [ - "docling" - ]; - - disabledTests = [ - "test_e2e_pdfs_conversions" # AssertionError: ## TableFormer: Table Structure Understanding with Transf - "test_e2e_conversions" # RuntimeError: Tesseract is not available - - # AssertionError - # assert doc.export_to_markdown() == pair[1], f"Error in case {idx}" - "test_ordered_lists" - - # AssertionError: export to md - "test_e2e_html_conversions" - - # AssertionError: assert 'Unordered li...d code block:' == 'Unordered li...d code block:' - "test_convert_valid" - - # AssertionError: Markdown file mismatch against groundtruth pftaps057006474.md - "test_patent_groundtruth" - - # huggingface_hub.errors.LocalEntryNotFoundError: An error happened - "test_cli_convert" - "test_code_and_formula_conversion" - "test_picture_classifier" - "test_convert_path" - "test_convert_stream" - "test_compare_legacy_output" - "test_ocr_coverage_threshold" - "test_formula_conversion_with_page_range" - - # requires network access - "test_page_range" - "test_parser_backends" - "test_confidence" - "test_e2e_webp_conversions" - "test_asr_pipeline_conversion" - "test_threaded_pipeline" - "test_pipeline_comparison" - - # AssertionError: pred_itxt==true_itxt - "test_e2e_valid_csv_conversions" + versionCheckHook ]; meta = { - description = "Get your documents ready for gen AI"; + description = "SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications"; homepage = "https://github.com/DS4SD/docling"; - changelog = "https://github.com/DS4SD/docling/blob/${src.tag}/CHANGELOG.md"; + changelog = "https://github.com/DS4SD/docling/blob/${finalAttrs.src.tag}/CHANGELOG.md"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ happysalada ]; mainProgram = "docling"; }; -} +}) diff --git a/pkgs/development/python-modules/rapidocr/default.nix b/pkgs/development/python-modules/rapidocr/default.nix index 0c2ce13959a7..2d42c4ac615b 100644 --- a/pkgs/development/python-modules/rapidocr/default.nix +++ b/pkgs/development/python-modules/rapidocr/default.nix @@ -3,11 +3,11 @@ buildPythonPackage, fetchFromGitHub, - fetchpatch, - fetchzip, - replaceVars, + fetchurl, + linkFarm, setuptools, + setuptools-scm, colorlog, pyclipper, opencv-python, @@ -23,75 +23,55 @@ requests, }: let - version = "3.8.1"; + version = "3.9.2"; + + # The models bundled into the wheel are the ones selected by the defaults in + # `python/rapidocr/config.yaml`. Run `python tools/prepare_wheel_assets.py --skip-manifest-in` + # from `python/` after a bump to check whether that selection changed; it prints the resolved + # URLs and their checksums. + models = linkFarm "rapidocr-models" { + "PP-OCRv6_det_small.onnx" = fetchurl { + url = "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v${version}/onnx/PP-OCRv6/det/PP-OCRv6_det_small.onnx"; + hash = "sha256-CQ8Eq82dmnSYvE6/Z35Mub3OH+QZfdt+Up8e9E4f+U8="; + }; + "ch_ppocr_mobile_v2.0_cls_mobile.onnx" = fetchurl { + url = "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v${version}/onnx/PP-OCRv4/cls/ch_ppocr_mobile_v2.0_cls_mobile.onnx"; + hash = "sha256-5HrO32YyMPiGP/GrDmTdLYK4OPzrWVcUbasYWonWIVw="; + }; + "PP-OCRv6_rec_small.onnx" = fetchurl { + url = "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v${version}/onnx/PP-OCRv6/rec/PP-OCRv6_rec_small.onnx"; + hash = "sha256-bzJyRrUDiPPBdq4wS9lXZ+ptwMmukhU++MviELPBSIQ="; + }; + }; +in +buildPythonPackage (finalAttrs: { + pname = "rapidocr"; + inherit version; + pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "RapidAI"; repo = "RapidOCR"; - tag = "v${version}"; - hash = "sha256-keAR7H/qn0Q+Vo0usp69dWZO5QWB60EgU7d9vspQ+2w="; + tag = "v${finalAttrs.version}"; + hash = "sha256-EdiYf8cM1+byEECK+3sRc7MC075BQV/PKIf/fC2Y1Lc="; }; - models = - fetchzip { - url = "https://github.com/RapidAI/RapidOCR/releases/download/v1.1.0/required_for_whl_v1.3.0.zip"; - hash = "sha256-j/0nzyvu/HfNTt5EZ+2Phe5dkyPOdQw/OZTz0yS63aA="; - stripRoot = false; - } - + "/required_for_whl_v1.3.0/resources/models"; -in -buildPythonPackage { - pname = "rapidocr"; - inherit version src; - pyproject = true; + sourceRoot = "${finalAttrs.src.name}/python"; - sourceRoot = "${src.name}/python"; + # Upstream ships an empty `models` directory and downloads the models at runtime. Pre-populate it + # so that the default configuration works without network access. + postPatch = '' + ln -s ${models}/* rapidocr/models + ''; - # HACK: - # Upstream uses a very unconventional structure to organize the packages, and we have to coax the - # existing infrastructure to work with it. - # See https://github.com/RapidAI/RapidOCR/blob/02829ef986bc2a5c4f33e9c45c9267bcf2d07a1d/.github/workflows/gen_whl_to_pypi_rapidocr_ort.yml#L80-L92 - # for the "intended" way of building this package. - - # The setup.py supplied by upstream tries to determine the current version by - # fetching the latest version of the package from PyPI, and then bumping the version number. - # This is not allowed in the Nix build environment as we do not have internet access, - # hence we patch that out and get the version from the build environment directly. - patches = [ - (replaceVars ./setup-py-override-version-checking.patch { - inherit version; - }) - # Fix type error in Immich which is caused by passing null to Path() when model_root_dir is the default null - (fetchpatch { - url = "https://github.com/RapidAI/RapidOCR/commit/57dfac08d8de63c4c00d21a1ab14a4a3b5c01975.patch"; - stripLen = 1; - hash = "sha256-G49mTvBOm20BFOll4Pc0X397ZABT1tWMXd8nlDjBr7E="; - }) + build-system = [ + setuptools + setuptools-scm ]; - postPatch = '' - mkdir -p rapidocr/models - - ln -s ${models}/* rapidocr/models - - echo "from .rapidocr.main import RapidOCR, VisRes" > __init__.py - ''; - - # Upstream expects the source files to be under rapidocr/rapidocr - # instead of rapidocr for the wheel to build correctly. - preBuild = '' - mkdir rapidocr_t - mv rapidocr rapidocr_t - mv rapidocr_t rapidocr - ''; - - # Revert the above hack - postBuild = '' - mv rapidocr rapidocr_t - mv rapidocr_t/* . - ''; - - build-system = [ setuptools ]; + # setuptools-scm derives the version from the git metadata, which fetchFromGitHub does not keep + env.SETUPTOOLS_SCM_PRETEND_VERSION = finalAttrs.version; dependencies = [ colorlog @@ -116,11 +96,11 @@ buildPythonPackage { doCheck = false; meta = { - changelog = "https://github.com/RapidAI/RapidOCR/releases/tag/${src.tag}"; description = "Cross platform OCR Library based on OnnxRuntime"; homepage = "https://github.com/RapidAI/RapidOCR"; + changelog = "https://github.com/RapidAI/RapidOCR/releases/tag/${finalAttrs.src.tag}"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ pluiedev ]; mainProgram = "rapidocr"; }; -} +}) diff --git a/pkgs/development/python-modules/rapidocr/setup-py-override-version-checking.patch b/pkgs/development/python-modules/rapidocr/setup-py-override-version-checking.patch deleted file mode 100644 index f940c1c29dc1..000000000000 --- a/pkgs/development/python-modules/rapidocr/setup-py-override-version-checking.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/setup.py b/setup.py -index 16938a4..5167972 100644 ---- a/setup.py -+++ b/setup.py -@@ -6,7 +6,6 @@ from pathlib import Path - from typing import List, Union - - import setuptools --from get_pypi_latest_version import GetPyPiLatestVersion - - - def read_txt(txt_path: Union[Path, str]) -> List[str]: -@@ -25,20 +24,7 @@ def get_readme(): - - - MODULE_NAME = "rapidocr" -- --obtainer = GetPyPiLatestVersion() --try: -- latest_version = obtainer(MODULE_NAME) --except Exception as e: -- latest_version = "0.0.0" --VERSION_NUM = obtainer.version_add_one(latest_version, add_patch=True) -- --if len(sys.argv) > 2: -- match_str = " ".join(sys.argv[2:]) -- matched_versions = obtainer.extract_version(match_str) -- if matched_versions: -- VERSION_NUM = matched_versions --sys.argv = sys.argv[:2] -+VERSION_NUM = "@version@" - - project_urls = { - "Documentation": "https://rapidai.github.io/RapidOCRDocs", diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d65a5f79f723..04ab0065e256 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5024,6 +5024,8 @@ self: super: with self; { dockerspawner = callPackage ../development/python-modules/dockerspawner { }; + doclang = callPackage ../development/python-modules/doclang { }; + docling = callPackage ../development/python-modules/docling { }; docling-core = callPackage ../development/python-modules/docling-core { }; @@ -5040,6 +5042,8 @@ self: super: with self; { docling-serve = callPackage ../development/python-modules/docling-serve { }; + docling-slim = callPackage ../development/python-modules/docling-slim { }; + docloud = callPackage ../development/python-modules/docloud { }; docopt = callPackage ../development/python-modules/docopt { };