Merge #236131: apache-airflow: 2.5.1 -> 2.6.0, remove from python3Packages

...into release-23.05

pkgs/development/python-modules/rich-argparse/default.nix

@@ -0,0 +1,41 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, hatchling
+, rich
+, pytestCheckHook
+}:
+
+buildPythonPackage rec {
+  pname = "rich-argparse";
+  version = "1.1.0";
+  format = "pyproject";
+
+  src = fetchFromGitHub {
+    owner = "hamdanal";
+    repo = "rich-argparse";
+    rev = "v${version}";
+    hash = "sha256-Rnv4A9pZ5VHpNjrWnsKyxQ4ISCLjIUu3tbbOzP4uFuw=";
+  };
+
+  propagatedBuildInputs = [
+    hatchling
+    rich
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "rich_argparse"
+  ];
+
+  meta = with lib; {
+    description = "Format argparse help output using rich.";
+    homepage = "https://github.com/hamdanal/rich-argparse";
+    changelog = "https://github.com/hamdanal/rich-argparse/blob/v${version}/CHANGELOG.md";
+    license = licenses.mit;
+    maintainers = with maintainers; [ graham33 ];
+  };
+}

pkgs/servers/apache-airflow/default.nix

@@ -0,0 +1,76 @@
+{ lib
+, fetchFromGitHub
+, fetchPypi
+, python3
+}:
+
+let
+  python = python3.override {
+    packageOverrides = pySelf: pySuper: {
+      # flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x
+      # https://github.com/dpgaspar/Flask-AppBuilder/issues/2038
+      flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs (o: rec {
+        version = "4.3.1";
+        src = fetchPypi {
+          pname = "Flask-AppBuilder";
+          inherit version;
+          hash = "sha256-FP92HEGOsufHtaIySqDiScD3QUu3iQhWdtvkOecUvuI=";
+        };
+
+        propagatedBuildInputs = o.propagatedBuildInputs ++ [
+          pySelf.flask-limiter
+        ];
+
+        meta.broken = false;
+      });
+      # a knock-on effect from overriding the sqlalchemy version
+      flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs (o: {
+        src = fetchPypi {
+          pname = "Flask-SQLAlchemy";
+          version = "2.5.1";
+          hash = "sha256-K9pEtD58rLFdTgX/PMH4vJeTbMRkYjQkECv8LDXpWRI=";
+        };
+        format = "setuptools";
+      });
+      # apache-airflow doesn't work with sqlalchemy 2.x
+      # https://github.com/apache/airflow/issues/28723
+      sqlalchemy = pySuper.sqlalchemy.overridePythonAttrs (o: rec {
+        version = "1.4.48";
+        src = fetchFromGitHub {
+          owner = "sqlalchemy";
+          repo = "sqlalchemy";
+          rev = "refs/tags/rel_${lib.replaceStrings [ "." ] [ "_" ] version}";
+          hash = "sha256-qyD3uoxEnD2pdVvwpUlSqHB3drD4Zg/+ov4CzLFIlLs=";
+        };
+      });
+      # a version bump required by airflow 2.6.0 that missed the nixos
+      # 23.05 deadline
+      python-daemon = pySuper.python-daemon.overridePythonAttrs (o: rec {
+        version = "3.0.1";
+        src = fetchPypi {
+          inherit version;
+          pname = o.pname;
+          sha256 = "sha256-bFdFI3L36v9Ak0ocA60YJr9eeTVY6H/vSRMeZGS02uU=";
+        };
+
+        patches = [];
+        disabledTestPaths = [];
+
+        pythonImportsCheck = lib.remove "daemon.runner" o.pythonImportsCheck;
+      });
+
+      apache-airflow = pySelf.callPackage ./python-package.nix { };
+    };
+  };
+in
+# See note in ./python-package.nix for
+# instructions on manually testing the web UI
+with python.pkgs; (toPythonApplication apache-airflow).overrideAttrs (_:{
+  # Provide access to airflow's modified python package set
+  # for the cases where external scripts need to import
+  # airflow modules, though *caveat emptor* because many of
+  # these packages will not be built by hydra and many will
+  # not work at all due to the unexpected version overrides
+  # here.
+  passthru.pythonPackages = python.pkgs;
+})

pkgs/servers/apache-airflow/package.json

@@ -9,6 +9,7 @@
     "build": "NODE_ENV=production webpack --progress --mode production",
     "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc",
     "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc",
+    "format": "yarn prettier --write .",
     "generate-api-types": "npx openapi-typescript \"../api_connexion/openapi/v1.yaml\" --output static/js/types/api-generated.ts && node alias-rest-types.js static/js/types/api-generated.ts"
   },
   "author": "Apache",
@@ -37,13 +38,14 @@
     "@babel/preset-typescript": "^7.17.12",
     "@testing-library/jest-dom": "^5.16.0",
     "@testing-library/react": "^13.0.0",
+    "@types/color": "^3.0.3",
     "@types/react": "^18.0.12",
     "@types/react-dom": "^18.0.5",
     "@types/react-table": "^7.7.12",
     "@typescript-eslint/eslint-plugin": "^5.13.0",
     "@typescript-eslint/parser": "^5.0.0",
     "babel-jest": "^27.3.1",
-    "babel-loader": "^8.1.0",
+    "babel-loader": "^9.1.0",
     "clean-webpack-plugin": "^3.0.0",
     "copy-webpack-plugin": "^6.0.3",
     "css-loader": "5.2.7",
@@ -51,8 +53,9 @@
     "eslint": "^8.6.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^17.0.0",
+    "eslint-config-prettier": "^8.6.0",
     "eslint-plugin-html": "^6.0.2",
-    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-jsx-a11y": "^6.5.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^4.2.1",
@@ -67,14 +70,16 @@
     "moment-locales-webpack-plugin": "^1.2.0",
     "nock": "^13.2.4",
     "openapi-typescript": "^5.4.1",
+    "prettier": "^2.8.4",
     "style-loader": "^1.2.1",
     "stylelint": "^13.6.1",
+    "stylelint-config-prettier": "^9.0.5",
     "stylelint-config-standard": "^20.0.0",
     "terser-webpack-plugin": "<5.0.0",
     "typescript": "^4.6.3",
     "url-loader": "4.1.0",
     "web-worker": "^1.2.0",
-    "webpack": "^5.73.0",
+    "webpack": "^5.76.0",
     "webpack-cli": "^4.0.0",
     "webpack-license-plugin": "^4.2.1",
     "webpack-manifest-plugin": "^4.0.0"
@@ -93,7 +98,9 @@
     "camelcase-keys": "^7.0.0",
     "chakra-react-select": "^4.0.0",
     "codemirror": "^5.59.1",
+    "color": "^4.2.3",
     "d3": "^3.4.4",
+    "d3-selection": "^3.0.0",
     "d3-shape": "^2.1.0",
     "d3-tip": "^0.9.1",
     "dagre-d3": "^0.6.4",
@@ -105,16 +112,27 @@
     "jquery": ">=3.5.0",
     "jshint": "^2.13.4",
     "lodash": "^4.17.21",
-    "moment-timezone": "^0.5.35",
+    "moment-timezone": "^0.5.43",
     "nvd3": "^1.8.6",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
     "react-icons": "^4.3.1",
+    "react-json-view": "^1.21.3",
+    "react-markdown": "^8.0.4",
     "react-query": "^3.39.1",
     "react-router-dom": "^6.3.0",
     "react-table": "^7.8.0",
+    "react-textarea-autosize": "^8.3.4",
+    "reactflow": "^11.4.0",
     "redoc": "^2.0.0-rc.72",
+    "remark-gfm": "^3.0.1",
+    "swagger-ui-dist": "4.1.3",
+    "tsconfig-paths": "^3.14.2",
     "type-fest": "^2.17.0",
-    "url-search-params-polyfill": "^8.1.0"
+    "url-search-params-polyfill": "^8.1.0",
+    "validator": "^13.9.0"
+  },
+  "resolutions": {
+    "d3-color": "^3.1.0"
   }
 }

pkgs/servers/apache-airflow/python-package.nix

@@ -5,6 +5,7 @@
 , fetchFromGitHub
 , alembic
 , argcomplete
+, asgiref
 , attrs
 , blinker
 , cached-property
@@ -48,6 +49,7 @@
 , pathspec
 , pendulum
 , psutil
+, pydantic
 , pygments
 , pyjwt
 , python-daemon
@@ -58,6 +60,7 @@
 , pythonOlder
 , pyyaml
 , rich
+, rich-argparse
 , setproctitle
 , sqlalchemy
 , sqlalchemy-jsonfield
@@ -68,8 +71,10 @@
 , typing-extensions
 , unicodecsv
 , werkzeug
-, pytestCheckHook
 , freezegun
+, pytest-asyncio
+, pytestCheckHook
+, time-machine
 , mkYarnPackage
 , writeScript
 
@@ -77,7 +82,7 @@
 , enabledProviders ? []
 }:
 let
-  version = "2.5.1";
+  version = "2.6.0";
 
   airflow-src = fetchFromGitHub rec {
     owner = "apache";
@@ -86,7 +91,7 @@ let
     # Download using the git protocol rather than using tarballs, because the
     # GitHub archive tarballs don't appear to include tests
     forceFetchGit = true;
-    hash = "sha256-BuJfE6SONTNonUvacOAIdZe0QicdBtx7k186TJZpQOs=";
+    hash = "sha256-CsLOj68+tSOI7drZq6QH+C2EB/9trg5cFAYAQp/06m8=";
   };
 
   # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
@@ -141,6 +146,7 @@ buildPythonPackage rec {
   propagatedBuildInputs = [
     alembic
     argcomplete
+    asgiref
     attrs
     blinker
     cached-property
@@ -183,6 +189,7 @@ buildPythonPackage rec {
     pathspec
     pendulum
     psutil
+    pydantic
     pygments
     pyjwt
     python-daemon
@@ -192,6 +199,7 @@ buildPythonPackage rec {
     python3-openid
     pyyaml
     rich
+    rich-argparse
     setproctitle
     sqlalchemy
     sqlalchemy-jsonfield
@@ -212,7 +220,9 @@ buildPythonPackage rec {
 
   nativeCheckInputs = [
     freezegun
+    pytest-asyncio
     pytestCheckHook
+    time-machine
   ];
 
   # By default, source code of providers is included but unusable due to missing
@@ -223,7 +233,7 @@ buildPythonPackage rec {
   postPatch = ''
     substituteInPlace setup.cfg \
       --replace "colorlog>=4.0.2, <5.0" "colorlog" \
-      --replace "flask-appbuilder==4.1.4" "flask-appbuilder>=4.1.3" \
+      --replace "flask-appbuilder==4.3.0" "flask-appbuilder>=4.3.0" \
       --replace "pathspec~=0.9.0" "pathspec"
   '' + lib.optionalString stdenv.isDarwin ''
     # Fix failing test on Hydra
@@ -281,9 +291,6 @@ buildPythonPackage rec {
     cd ./pkgs/development/python-modules/apache-airflow
     curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
     curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
-    # Revert this commit which seems to break with our version of yarn
-    # https://github.com/apache/airflow/commit/b9e133e40c2848b0d555051a99bf8d2816fd28a7
-    patch -p3 < 0001-Revert-fix-yarn-warning-from-d3-color-27139.patch
     yarn2nix > yarn.nix
 
     # update provider dependencies
@@ -293,7 +300,7 @@ buildPythonPackage rec {
   # Note on testing the web UI:
   # You can (manually) test the web UI as follows:
   #
-  #   nix shell .#python3Packages.apache-airflow
+  #   nix shell .#apache-airflow
   #   airflow db reset  # WARNING: this will wipe any existing db state you might have!
   #   airflow db init
   #   airflow standalone

pkgs/servers/apache-airflow/update-providers.py

@@ -13,12 +13,13 @@ from urllib.request import urlopen
 from urllib.error import HTTPError
 import yaml
 
-PKG_SET = "pkgs.python3Packages"
+PKG_SET = "apache-airflow.pythonPackages"
 
 # If some requirements are matched by multiple or no Python packages, the
 # following can be used to choose the correct one
 PKG_PREFERENCES = {
     "dnspython": "dnspython",
+    "elasticsearch-dsl": "elasticsearch-dsl",
     "google-api-python-client": "google-api-python-client",
     "psycopg2-binary": "psycopg2",
     "requests_toolbelt": "requests-toolbelt",

pkgs/top-level/all-packages.nix

@@ -1436,7 +1436,7 @@ with pkgs;
 
   airgeddon = callPackage ../tools/networking/airgeddon { };
 
-  apache-airflow = with python3.pkgs; toPythonApplication apache-airflow;
+  apache-airflow = callPackage ../servers/apache-airflow { };
 
   apachetomcatscanner = callPackage ../tools/security/apachetomcatscanner { };
 

pkgs/top-level/python-packages.nix

@@ -528,8 +528,6 @@ self: super: with self; {
 
   aocd = callPackage ../development/python-modules/aocd { };
 
-  apache-airflow = callPackage ../development/python-modules/apache-airflow { };
-
   apache-beam = callPackage ../development/python-modules/apache-beam { };
 
   apcaccess = callPackage ../development/python-modules/apcaccess { };
@@ -10554,6 +10552,8 @@ self: super: with self; {
 
   rich = callPackage ../development/python-modules/rich { };
 
+  rich-argparse = callPackage ../development/python-modules/rich-argparse { };
+
   rich-argparse-plus = callPackage ../development/python-modules/rich-argparse-plus { };
 
   rich-click = callPackage ../development/python-modules/rich-click { };