Merge master into staging-next

This brings thousands of rebuilds.
Hydra nixpkgs: ?compare=1597944

pkgs/development/python-modules/btrfs/default.nix

@@ -0,0 +1,22 @@
+{ stdenv
+, buildPythonPackage
+, fetchPypi
+}:
+
+buildPythonPackage rec {
+  pname = "btrfs";
+  version = "11";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1w92sj47wy53ygz725xr613k32pk5khi0g9lrpp6img871241hrx";
+  };
+
+  meta = with stdenv.lib; {
+    description = "Inspect btrfs filesystems";
+    homepage = "https://github.com/knorrie/python-btrfs";
+    license = licenses.lgpl3Plus;
+    platforms = platforms.linux;
+    maintainers = [ maintainers.evils ];
+  };
+}

pkgs/development/python-modules/canmatrix/default.nix

@@ -4,6 +4,7 @@
 , pythonOlder
 , attrs
 , bitstruct
+, click
 , future
 , pathlib2
 , typing
@@ -17,20 +18,21 @@
 
 buildPythonPackage rec {
   pname = "canmatrix";
-  version = "0.8";
+  version = "0.9.1";
 
   # uses fetchFromGitHub as PyPi release misses test/ dir
   src = fetchFromGitHub {
     owner = "ebroecker";
     repo = pname;
     rev = version;
-    sha256 = "1wzflapyj2j4xsi7d7gfmznmxbgr658n092xyq9nac46rbhpcphg";
+    sha256 = "129lcchq45h8wqjvvn0rwpbmih4m0igass2cx7a21z94li97hcia";
   };
 
   propagatedBuildInputs = [
     # required
     attrs
     bitstruct
+    click
     future
     pathlib2
     # optional
@@ -41,12 +43,18 @@ buildPythonPackage rec {
     pyyaml
   ] ++ lib.optional (pythonOlder "3.5") typing;
 
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "version = versioneer.get_version()" "version = \"${version}\""
+  '';
+
   checkInputs = [
     pytest
   ];
 
+  # long_envvar_name_imports requires stable key value pair ordering
   checkPhase = ''
-    pytest -s src/canmatrix
+    pytest -s src/canmatrix -k 'not long_envvar_name_imports'
   '';
 
   meta = with lib; {

pkgs/development/python-modules/canopen/default.nix

@@ -1,33 +1,19 @@
 { lib
 , buildPythonPackage
-, fetchFromGitHub
+, fetchPypi
 , nose
 , can
 , canmatrix }:
 
-buildPythonPackage {
+buildPythonPackage rec {
   pname = "canopen";
-  version = "0.5.1";
+  version = "1.1.0";
 
-  # use fetchFromGitHub until version containing test/sample.eds
-  # is available on PyPi
-  # https://github.com/christiansandberg/canopen/pull/57
-
-  src = fetchFromGitHub {
-    owner = "christiansandberg";
-    repo = "canopen";
-    rev = "b20575d84c3aef790fe7c38c5fc77601bade0ea4";
-    sha256 = "1qg47qrkyvyxiwi13sickrkk89jp9s91sly2y90bz0jhws2bxh64";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0fqa4p3qg7800fykib1x264gizhhmb6dz2hajgwr0hxf5xa19wdl";
   };
 
-  #src = fetchPypi {
-  #  inherit pname version;
-  #  sha256 = "0806cykarpjb9ili3mf82hsd9gdydbks8532nxgz93qzg4zdbv2g";
-  #};
-
-  # test_pdo failure https://github.com/christiansandberg/canopen/issues/58
-  doCheck = false;
-
   propagatedBuildInputs =
     [ can
       canmatrix

pkgs/development/python-modules/debugpy/default.nix

@@ -0,0 +1,68 @@
+{ stdenv, buildPythonPackage, fetchFromGitHub
+, substituteAll, gdb
+, colorama, django, flask, gevent, psutil, pytest
+, pytest-timeout, pytest_xdist, requests
+, isPy27
+}:
+
+buildPythonPackage rec {
+  pname = "debugpy";
+  version = "1.0.0b12";
+
+  src = fetchFromGitHub {
+    owner = "Microsoft";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0sz33aq5qldl7kh4qjf5w3d08l9s77ipcj4i9wfklj8f6vf9w1wh";
+  };
+
+  patches = [
+    # Hard code GDB path (used to attach to process)
+    (substituteAll {
+      src = ./hardcode-gdb.patch;
+      inherit gdb;
+    })
+
+    # Fix importing debugpy in:
+    # - test_nodebug[module-launch(externalTerminal)]
+    # - test_nodebug[module-launch(integratedTerminal)]
+    #
+    # NOTE: The import failures seen in these tests without the patch
+    # will be seen if a user "installs" debugpy by adding it to PYTHONPATH.
+    # To avoid this issue, debugpy should be installed using python.withPackages:
+    # python.withPackages (ps: with ps; [ debugpy ])
+    ./fix-test-pythonpath.patch
+  ];
+
+  # Remove pre-compiled "attach" libraries and recompile for host platform
+  # Compile flags taken from linux_and_mac/compile_linux.sh & linux_and_mac/compile_mac.sh
+  preBuild = ''(
+    set -x
+    cd src/debugpy/_vendored/pydevd/pydevd_attach_to_process
+    rm *.so *.dylib *.dll *.exe *.pdb
+    ${stdenv.cc}/bin/c++ linux_and_mac/attach.cpp -Ilinux_and_mac -fPIC -nostartfiles ${{
+      "x86_64-linux"  = "-shared -m64 -o attach_linux_amd64.so";
+      "i686-linux"    = "-shared -m32 -o attach_linux_x86.so";
+      "x86_64-darwin" = "-std=c++11 -lc -D_REENTRANT -dynamiclib -arch x86_64 -o attach_x86_64.dylib";
+      "i686-darwin"   = "-std=c++11 -lc -D_REENTRANT -dynamiclib -arch i386 -o attach_x86.dylib";
+    }.${stdenv.hostPlatform.system}}
+  )'';
+
+  checkInputs = [
+    colorama django flask gevent psutil pytest
+    pytest-timeout pytest_xdist requests
+  ];
+
+  # Override default arguments in pytest.ini
+  checkPhase = "pytest --timeout 0 -n $NIX_BUILD_CORES"
+               # gevent fails to import zope.interface with Python 2.7
+               + stdenv.lib.optionalString isPy27 " -k 'not test_gevent'";
+
+  meta = with stdenv.lib; {
+    description = "An implementation of the Debug Adapter Protocol for Python";
+    homepage = "https://github.com/microsoft/debugpy";
+    license = licenses.mit;
+    maintainers = with maintainers; [ metadark ];
+    platforms = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "i686-darwin" ];
+  };
+}

pkgs/development/python-modules/debugpy/fix-test-pythonpath.patch

@@ -0,0 +1,12 @@
+diff --git a/tests/debug/session.py b/tests/debug/session.py
+index 2b39106..6d45a10 100644
+--- a/tests/debug/session.py
++++ b/tests/debug/session.py
+@@ -625,6 +625,7 @@ class Session(object):
+         if "PYTHONPATH" in self.config.env:
+             # If specified, launcher will use it in lieu of PYTHONPATH it inherited
+             # from the adapter when spawning debuggee, so we need to adjust again.
++            self.config.env.prepend_to("PYTHONPATH", os.environ["PYTHONPATH"])
+             self.config.env.prepend_to("PYTHONPATH", DEBUGGEE_PYTHONPATH.strpath)
+         return self._request_start("launch")
+ 

pkgs/development/python-modules/debugpy/hardcode-gdb.patch

@@ -0,0 +1,13 @@
+diff --git a/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/add_code_to_python_process.py b/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/add_code_to_python_process.py
+index 6d031b4..ecf21f2 100644
+--- a/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/add_code_to_python_process.py
++++ b/src/debugpy/_vendored/pydevd/pydevd_attach_to_process/add_code_to_python_process.py
+@@ -293,7 +293,7 @@ def run_python_code_linux(pid, python_code, connect_debugger_tracing=False, show
+     is_debug = 0
+     # Note that the space in the beginning of each line in the multi-line is important!
+     cmd = [
+-        'gdb',
++        '@gdb@/bin/gdb',
+         '--nw',  # no gui interface
+         '--nh',  # no ~/.gdbinit
+         '--nx',  # no .gdbinit

pkgs/development/python-modules/denonavr/default.nix

@@ -1,18 +1,18 @@
-{ lib, buildPythonPackage, fetchFromGitHub, isPy27, requests
+{ lib, buildPythonPackage, fetchFromGitHub, isPy27, requests, netifaces
 , pytest, testtools, requests-mock }:
 
 buildPythonPackage rec {
   pname = "denonavr";
-  version = "0.8.1";
+  version = "0.9.3";
 
   src = fetchFromGitHub {
     owner = "scarface-4711";
     repo = "denonavr";
     rev = version;
-    sha256 = "12g9w5674fmyf3f4plbhvpxpyhzw32pzwl0hvwswzrc2823xl6vx";
+    sha256 = "0s8v918n6xn44r2mrq5hqbf0znpz64clq7a1jakkgz9py8bi6vnn";
   };
 
-  propagatedBuildInputs = [ requests ];
+  propagatedBuildInputs = [ requests netifaces ];
 
   doCheck = !isPy27;
   checkInputs = [ pytest testtools requests-mock ];

pkgs/development/python-modules/executing/default.nix

@@ -0,0 +1,20 @@
+{ lib, buildPythonPackage, fetchzip, pytest, asttokens }:
+
+buildPythonPackage rec {
+  pname = "executing";
+  version = "0.4.3";
+
+  src = fetchzip {
+    url = "https://github.com/alexmojaki/executing/archive/v${version}.tar.gz";
+    sha256 = "1fqfc26nl703nsx2flzf7x4mgr3rpbd8pnn9c195rca648zhi3nh";
+  };
+
+  checkInputs = [ pytest asttokens ];
+
+  meta = with lib; {
+    description = "Get information about what a frame is currently doing, particularly the AST node being executed";
+    homepage = "https://github.com/alexmojaki/executing";
+    license = licenses.mit;
+    maintainers = with maintainers; [ renatoGarcia ];
+  };
+}

pkgs/development/python-modules/foxdot/default.nix

@@ -2,11 +2,11 @@
 
 buildPythonPackage rec {
   pname = "FoxDot";
-  version = "0.8.8";
+  version = "0.8.11";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "06y626kgaz1wn1qajlngihpvd4qz8m6lx6sknmjqhhrznyji58wi";
+    sha256 = "00yqpkv7cxwk301cyiwjzr9yfq8hpnhqyspw3z874ydrl3cmssdb";
   };
 
   propagatedBuildInputs = [ tkinter supercollider ];

pkgs/development/python-modules/icecream/default.nix

@@ -0,0 +1,22 @@
+{ lib, buildPythonPackage, fetchPypi
+, asttokens, colorama, executing, pygments
+}:
+
+buildPythonPackage rec {
+  pname = "icecream";
+  version = "2.0.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "16kpixl43nrn093cvkmxiq2dzd9xc73zwzkmwp0rs7x01nji8kj3";
+  };
+
+  propagatedBuildInputs = [ asttokens colorama executing pygments ];
+
+  meta = with lib; {
+    description = "A little library for sweet and creamy print debugging";
+    homepage = "https://github.com/gruns/icecream";
+    license = licenses.mit;
+    maintainers = with maintainers; [ renatoGarcia ];
+  };
+}

pkgs/development/python-modules/jc/default.nix

@@ -9,14 +9,14 @@
 
 buildPythonPackage rec {
   pname = "jc";
-  version = "1.11.6";
+  version = "1.11.8";
   disabled = isPy27;
 
   src = fetchFromGitHub {
     owner = "kellyjonbrazil";
     repo = "jc";
     rev = "v${version}";
-    sha256 = "0jyygq7zmam7yriiv5j4d6mpjdi2p3p7d53bn3qwfzkh4ifsbfan";
+    sha256 = "0rkckbgm04ql4r48wjgljfiqvsz36n99yqcpcyna8lvlm8h4nmwa";
   };
 
   propagatedBuildInputs = [ ruamel_yaml xmltodict pygments ];

pkgs/development/python-modules/matrix-nio/default.nix

@@ -20,13 +20,13 @@
 
 buildPythonPackage rec {
   pname = "matrix-nio";
-  version = "0.12.0";
+  version = "0.14.1";
 
   src = fetchFromGitHub {
     owner = "poljar";
     repo = "matrix-nio";
     rev = version;
-    sha256 = "1d4jhl2xjjp31yjs4zz2cfkvzbi2vr5bsrp07s6nhyc18azxr7ba";
+    sha256 = "0mgb9m3298jvw3wa051zn7vp1m8qriys3ps0qn3sq54fndljgg5k";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/sparse/default.nix

@@ -2,6 +2,7 @@
 , buildPythonPackage
 , fetchPypi
 , isPy3k
+, dask
 , numpy
 , scipy
 , numba
@@ -19,7 +20,7 @@ buildPythonPackage rec {
     sha256 = "ffbca00a53f938e4f04230f582b210440efb54d74d60af1d1ced3864f61677ac";
   };
 
-  checkInputs = [ pytest ];
+  checkInputs = [ pytest dask ];
   propagatedBuildInputs = [
     numpy
     scipy

pkgs/development/python-modules/tokenizers/default.nix

@@ -0,0 +1,121 @@
+{ stdenv
+, rustPlatform
+, fetchFromGitHub
+, fetchurl
+, maturin
+, pipInstallHook
+, pytest
+, python
+, requests
+}:
+
+let
+  robertaVocab = fetchurl {
+    url = "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json";
+    sha256 = "0m86wpkfb2gdh9x9i9ng2fvwk1rva4p0s98xw996nrjxs7166zwy";
+  };
+  robertaMerges = fetchurl {
+    url = "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt";
+    sha256 = "1idd4rvkpqqbks51i2vjbd928inw7slij9l4r063w3y5fd3ndq8w";
+  };
+  bertVocab = fetchurl {
+    url = "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt";
+    sha256 = "18rq42cmqa8zanydsbzrb34xwy4l6cz1y900r4kls57cbhvyvv07";
+  };
+  openaiVocab = fetchurl {
+    url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json";
+    sha256 = "0y40gc9bixj5rxv674br1rxmxkd3ly29p80x1596h8yywwcrpx7x";
+  };
+  openaiMerges = fetchurl {
+    url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt";
+    sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f";
+  };
+in rustPlatform.buildRustPackage rec {
+  pname = "tokenizers";
+  version = "0.8.0";
+
+  src = fetchFromGitHub {
+    owner = "huggingface";
+    repo = pname;
+    rev = "python-v${version}";
+    sha256 = "0f5r1wm5ybyk3jvihj1g98y7ihq0iklg0pwkaa11pk1gv0k869w3";
+  };
+
+  cargoSha256 = "131bvf35q5n65mq6zws1rp5fn2qkfwfg9sbxi5y6if24n8fpdz4m";
+
+  sourceRoot = "source/bindings/python";
+
+  nativeBuildInputs = [
+    maturin
+    pipInstallHook
+  ];
+
+  propagatedBuildInputs = [
+    python
+  ];
+
+  # tokenizers uses pyo3, which requires Rust nightly.
+  RUSTC_BOOTSTRAP = 1;
+
+  doCheck = false;
+  doInstallCheck = true;
+
+  postUnpack = ''
+    # Add data files for tests, otherwise tests attempt network access.
+    mkdir $sourceRoot/tests/data
+    ( cd $sourceRoot/tests/data
+      ln -s ${robertaVocab} roberta-base-vocab.json
+      ln -s ${robertaMerges} roberta-base-merges.txt
+      ln -s ${bertVocab} bert-base-uncased-vocab.txt
+      ln -s ${openaiVocab} openai-gpt-vocab.json
+      ln -s ${openaiMerges} openai-gpt-merges.txt )
+  '';
+
+  postPatch = ''
+    # pyo3's build check verifies that Rust is a nightly
+    # version. Disable this check.
+    substituteInPlace $NIX_BUILD_TOP/$cargoDepsCopy/pyo3/build.rs \
+      --replace "check_rustc_version()?;" ""
+
+    # Patching the vendored dependency invalidates the file
+    # checksums, so remove them. This should be safe, since
+    # this is just a copy of the vendored dependencies and
+    # the integrity of the vendored dependencies is validated
+    # by cargoSha256.
+    sed -r -i 's|"files":\{[^}]+\}|"files":{}|' \
+      $NIX_BUILD_TOP/$cargoDepsCopy/pyo3/.cargo-checksum.json
+
+    # Maturin uses the crate name as the wheel name.
+    substituteInPlace Cargo.toml \
+      --replace "tokenizers-python" "tokenizers"
+  '';
+
+  buildPhase = ''
+    maturin build --release --manylinux off
+  '';
+
+  installPhase = ''
+    # Put the wheels where the pip install hook can find them.
+    install -Dm644 -t dist target/wheels/*.whl
+    pipInstallPhase
+  '';
+
+  installCheckInputs = [
+    pytest
+    requests
+  ];
+
+  installCheckPhase = ''
+    # Append paths, or the binding's tokenizer module will be
+    # used, since the test directories have __init__.py
+    pytest --import-mode=append
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = "https://github.com/huggingface/tokenizers";
+    description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ danieldk ];
+  };
+}

pkgs/development/python-modules/transformers/default.nix

@@ -1,41 +1,81 @@
 { buildPythonPackage
 , stdenv
 , fetchFromGitHub
-, sacremoses
-, requests
-, sentencepiece
 , boto3
-, tqdm
+, filelock
 , regex
+, requests
 , numpy
-, pytest
+, sacremoses
+, sentencepiece
+, timeout-decorator
+, tokenizers
+, tqdm
+, pytestCheckHook
 }:
 
 buildPythonPackage rec {
   pname = "transformers";
-  version = "2.2.1";
+  version = "3.0.1";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "v${version}";
-    sha256 = "1p8p3lhhiyk1xl9gpgq4vbchyz57v3w7hhvsj1r90zs3cckindl8";
+    sha256 = "1l8l82zi021sq5dnzlbjx3wx0n4yy7k96n3m2fr893y9lfkhhd8z";
   };
 
-  propagatedBuildInputs = [ numpy sacremoses requests sentencepiece boto3 tqdm regex ];
+  propagatedBuildInputs = [
+    boto3
+    filelock
+    numpy
+    regex
+    requests
+    sacremoses
+    sentencepiece
+    tokenizers
+    tqdm
+  ];
 
-  checkInputs = [ pytest ];
-  # pretrained tries to download from s3
-  checkPhase = ''
-    cd transformers # avoid importing local files
-    HOME=$TMPDIR pytest -k 'not pretrained_tokenizers'
+  checkInputs = [
+    pytestCheckHook
+    timeout-decorator
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "tokenizers == 0.8.0-rc4" "tokenizers>=0.8,<0.9"
   '';
 
+  preCheck = ''
+    export HOME="$TMPDIR"
+    cd tests
+  '';
+
+  # Disable tests that require network access.
+  disabledTests = [
+    "test_all_tokenizers"
+    "test_batch_encoding_is_fast"
+    "test_batch_encoding_pickle"
+    "test_config_from_model_shortcut"
+    "test_config_model_type_from_model_identifier"
+    "test_from_pretrained_use_fast_toggle"
+    "test_hf_api"
+    "test_outputs_can_be_shorter"
+    "test_outputs_not_longer_than_maxlen"
+    "test_pretokenized_tokenizers"
+    "test_tokenizer_equivalence_en_de"
+    "test_tokenizer_from_model_type"
+    "test_tokenizer_from_model_type"
+    "test_tokenizer_from_pretrained"
+    "test_tokenizer_identifier_with_correct_config"
+  ];
+
   meta = with stdenv.lib; {
     homepage = "https://github.com/huggingface/transformers";
     description = "State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch";
     license = licenses.asl20;
     platforms = [ "x86_64-linux" ];
-    maintainers = with maintainers; [ pashashocky ];
+    maintainers = with maintainers; [ danieldk pashashocky ];
   };
 }

pkgs/development/python-modules/zstandard/default.nix

@@ -3,19 +3,18 @@
 , fetchPypi
 , cffi
 , hypothesis
-, zstd
 }:
 
 buildPythonPackage rec {
   pname = "zstandard";
-  version = "0.13.0";
+  version = "0.14.0";
   
   src = fetchPypi {
     inherit pname version;
-    sha256 = "e5cbd8b751bd498f275b0582f449f92f14e64f4e03b5bf51c571240d40d43561";
+    sha256 = "0lkn7n3bfp7zip6hkqwkqwc8pxmhhs4rr699k77h51rfln6kjllh";
   };
   
-  propagatedBuildInputs = [ cffi zstd ];
+  propagatedBuildInputs = [ cffi ];
   
   checkInputs = [ hypothesis ];