From 99c0bf4bf08f098480160eea78d11b2e730982ff Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Wed, 27 Oct 2021 11:17:42 +0200 Subject: [PATCH 01/32] python3Packages.aionotion: 3.0.2 -> 2021.10.0 --- pkgs/development/python-modules/aionotion/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/aionotion/default.nix b/pkgs/development/python-modules/aionotion/default.nix index 459a06d57d4c..4e161ba6804b 100644 --- a/pkgs/development/python-modules/aionotion/default.nix +++ b/pkgs/development/python-modules/aionotion/default.nix @@ -13,7 +13,7 @@ buildPythonPackage rec { pname = "aionotion"; - version = "3.0.2"; + version = "2021.10.0"; format = "pyproject"; disabled = pythonOlder "3.6"; @@ -21,7 +21,7 @@ buildPythonPackage rec { owner = "bachya"; repo = pname; rev = version; - sha256 = "1328g2245h9gcrnzrbcxaxw78723d0skznrrj8k77fbijxdc4kwv"; + sha256 = "sha256-5bu2XveDi2e6lGdI/T+0apqXzcnqdztVtaHEpp+Vkzs="; }; nativeBuildInputs = [ poetry-core ]; From 9c73c4d06f7d9b871981c5ee62c2f3c0e84aa006 Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Wed, 27 Oct 2021 11:25:20 +0200 Subject: [PATCH 02/32] python3Packages.aioridwell: 0.2.0 -> 2021.10.0 --- pkgs/development/python-modules/aioridwell/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/aioridwell/default.nix b/pkgs/development/python-modules/aioridwell/default.nix index a11e06791f69..9f2c665f6754 100644 --- a/pkgs/development/python-modules/aioridwell/default.nix +++ b/pkgs/development/python-modules/aioridwell/default.nix @@ -17,7 +17,7 @@ buildPythonPackage rec { pname = "aioridwell"; - version = "0.2.0"; + version = "2021.10.0"; format = "pyproject"; disabled = pythonOlder "3.8"; @@ -26,7 +26,7 @@ buildPythonPackage rec { owner = "bachya"; repo = pname; rev = version; - sha256 = "0kzxh1igncql1rwh12n42qahlc94z0fsd344d1vgni50jg7i34v7"; + sha256 = "sha256-h89gfdZvk7H22xAczaPMscTYZu0YeFxvFfL6/Oz2cJw="; }; nativeBuildInputs = [ From 0ff42762ebd46ee186c832c17a6df696b3675d94 Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Wed, 27 Oct 2021 11:27:18 +0200 Subject: [PATCH 03/32] python3Packages.aiowatttime: 0.1.1 -> 2021.10.0 --- pkgs/development/python-modules/aiowatttime/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/aiowatttime/default.nix b/pkgs/development/python-modules/aiowatttime/default.nix index 39aa7974313c..67d21b7ebfab 100644 --- a/pkgs/development/python-modules/aiowatttime/default.nix +++ b/pkgs/development/python-modules/aiowatttime/default.nix @@ -12,7 +12,7 @@ buildPythonPackage rec { pname = "aiowatttime"; - version = "0.1.1"; + version = "2021.10.0"; format = "pyproject"; disabled = pythonOlder "3.8"; @@ -21,7 +21,7 @@ buildPythonPackage rec { owner = "bachya"; repo = pname; rev = version; - sha256 = "1614p5ca7x9ipz7dgwhiz83dfwn6hyliawa8pr2j9y2kn8cg2sdm"; + sha256 = "sha256-cWXhQMgRYBzOVgUQWONIwWFB5n/f0lqkSjUb9IoPwtI="; }; nativeBuildInputs = [ From 4080dff4f6bcf0a748d21a6b6d53c866e6eeec15 Mon Sep 17 00:00:00 2001 From: ajs124 Date: Sun, 24 Oct 2021 23:34:10 +0200 Subject: [PATCH 04/32] gcc8: 8.4.0 -> 8.5.0 --- pkgs/development/compilers/gcc/8/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/compilers/gcc/8/default.nix b/pkgs/development/compilers/gcc/8/default.nix index a6fd34c60cc3..01deb2727b85 100644 --- a/pkgs/development/compilers/gcc/8/default.nix +++ b/pkgs/development/compilers/gcc/8/default.nix @@ -54,7 +54,7 @@ with lib; with builtins; let majorVersion = "8"; - version = "${majorVersion}.4.0"; + version = "${majorVersion}.5.0"; inherit (stdenv) buildPlatform hostPlatform targetPlatform; @@ -90,7 +90,7 @@ stdenv.mkDerivation ({ src = fetchurl { url = "mirror://gcc/releases/gcc-${version}/gcc-${version}.tar.xz"; - sha256 = "1m1d3gfix56w4aq8myazzfffkl8bqcrx4jhhapnjf7qfs596w2p3"; + sha256 = "0l7d4m9jx124xsk6xardchgy2k5j5l2b15q322k31f0va4d8826k"; }; inherit patches; From 6366a136e3daa8b61e56cf7c53e07b4c97a64eba Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 28 Oct 2021 01:01:48 +0000 Subject: [PATCH 05/32] yubikey-manager-qt: 1.2.3 -> 1.2.4 --- pkgs/tools/misc/yubikey-manager-qt/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/misc/yubikey-manager-qt/default.nix b/pkgs/tools/misc/yubikey-manager-qt/default.nix index 5ef75778f254..0a5ccda2cedf 100644 --- a/pkgs/tools/misc/yubikey-manager-qt/default.nix +++ b/pkgs/tools/misc/yubikey-manager-qt/default.nix @@ -14,11 +14,11 @@ mkDerivation rec { pname = "yubikey-manager-qt"; - version = "1.2.3"; + version = "1.2.4"; src = fetchurl { url = "https://developers.yubico.com/${pname}/Releases/${pname}-${version}.tar.gz"; - sha256 = "sha256-54HvuJXjm846sBxwNHLmaBXvO24bbBDyK8YvY4I6LjY="; + sha256 = "sha256-PxHc7IeRsO+CPrNTofGypLLW8fSHDkcBqr75NwdlUyc="; }; nativeBuildInputs = [ From 8ff3aad7fe0016c919670524a91638ea7f35830b Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 28 Oct 2021 01:38:04 +0000 Subject: [PATCH 06/32] xscreensaver: 6.01 -> 6.02 --- pkgs/misc/screensavers/xscreensaver/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/misc/screensavers/xscreensaver/default.nix b/pkgs/misc/screensavers/xscreensaver/default.nix index ede38c6fa040..81f37b0bc41a 100644 --- a/pkgs/misc/screensavers/xscreensaver/default.nix +++ b/pkgs/misc/screensavers/xscreensaver/default.nix @@ -9,12 +9,12 @@ }: stdenv.mkDerivation rec { - version = "6.01"; + version = "6.02"; pname = "xscreensaver"; src = fetchurl { url = "https://www.jwz.org/${pname}/${pname}-${version}.tar.gz"; - sha256 = "sha256-CFSEZl2R9gtKHe2s2UvPm3Sw+wlrztyJ/xwkUWjlRzs="; + sha256 = "sha256-Xm1ssJAzrVYs/m1Gv5MS5EUfeUa+2KRnGqk0TfkZMYQ="; }; nativeBuildInputs = [ From b77b452186ea1b3918084083c2107d9f7af0ffdc Mon Sep 17 00:00:00 2001 From: TredwellGit Date: Thu, 28 Oct 2021 03:18:15 +0000 Subject: [PATCH 07/32] electron_13: 13.6.0 -> 13.6.1 https://github.com/electron/electron/releases/tag/v13.6.1 --- pkgs/development/tools/electron/default.nix | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkgs/development/tools/electron/default.nix b/pkgs/development/tools/electron/default.nix index c50f8ebc40d4..38f1827e5126 100644 --- a/pkgs/development/tools/electron/default.nix +++ b/pkgs/development/tools/electron/default.nix @@ -115,14 +115,14 @@ rec { headers = "1fvqkw08pync38ixi5cq4f8a108k2ajxpm1w2f8sn2hjph9kpbsd"; }; - electron_13 = mkElectron "13.6.0" { - armv7l-linux = "03141d99268b1e71b64fd663c678591a02544c99475967d511041ab2c5de262c"; - aarch64-linux = "309014cce9216e3b0a0015a7a31c33e39c63af0b7781e50d3e630fb1d7d3a4ee"; - x86_64-linux = "b7f88baa69ea6c06ca16e215b01ad673cd922d3a0c9239868ddd0072af4e035e"; - i686-linux = "b588e4fe82e6c37fe838a3cf592a2a4cf6166f26dad97dd7597818d7714ddc94"; - x86_64-darwin = "250228bc5a186601d831f86faf394fba732436ddec86c1ef70e4d3d7ff9342c7"; - aarch64-darwin = "fd3886830fa291db38cc770176ddb3167f8fb90065fcb124394b3ca32965108d"; - headers = "1rmx0gvcsvba24fsjv14vmmd52h1nzscgmf1g9i46kzc13y0hn4z"; + electron_13 = mkElectron "13.6.1" { + armv7l-linux = "c8bba8da0baf5cde3eb4823c801c228abfa7943c69131b3701c74e2b342e1813"; + aarch64-linux = "09a1ff29c33a23f19cc702e0147dee03cfe2acedcff6bfb63c7911184b871a1a"; + x86_64-linux = "bfc09dd2d591ad614c8d598dad6e13b76c3baf4f48773e7819c493b524a0bb1a"; + i686-linux = "1ea7c7d19759fa0ee0ddef68c09bcc1c57265436d3f5dab37dad3567f117f317"; + x86_64-darwin = "ce45f17f875d72e791999eaf30a1af39b6e9143e57a653e7f06cfa0bee9b985d"; + aarch64-darwin = "e2f82720acae3a03d7d4b3e7dcc7697b84d5bb69a63d087a7420ace2412e7a28"; + headers = "1bd87c74863w0sjs8gfxl62kjjscc56dmmw85vhwz01s4niksr02"; }; electron_14 = mkElectron "14.2.0" { From 40bad15b21f81b607f1ce54ef90d244e00b5c237 Mon Sep 17 00:00:00 2001 From: superherointj <5861043+superherointj@users.noreply.github.com> Date: Wed, 27 Oct 2021 09:10:44 -0300 Subject: [PATCH 08/32] ocamlPackages.dune-site: init 2.9.1 --- .../ocaml-modules/dune-site/default.nix | 19 +++++++++++++++++++ pkgs/top-level/ocaml-packages.nix | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 pkgs/development/ocaml-modules/dune-site/default.nix diff --git a/pkgs/development/ocaml-modules/dune-site/default.nix b/pkgs/development/ocaml-modules/dune-site/default.nix new file mode 100644 index 000000000000..27dcfe660ef8 --- /dev/null +++ b/pkgs/development/ocaml-modules/dune-site/default.nix @@ -0,0 +1,19 @@ +{ lib, buildDunePackage, dune_2, dune-private-libs }: + +buildDunePackage rec { + pname = "dune-site"; + inherit (dune_2) src version patches; + + useDune2 = true; + + dontAddPrefix = true; + + propagatedBuildInputs = [ dune-private-libs ]; + + meta = with lib; { + description = "A library for embedding location information inside executable and libraries"; + inherit (dune_2.meta) homepage; + maintainers = with lib.maintainers; [ superherointj ]; + license = licenses.mit; + }; +} diff --git a/pkgs/top-level/ocaml-packages.nix b/pkgs/top-level/ocaml-packages.nix index c9da008e8f5a..13ed6c5a37fd 100644 --- a/pkgs/top-level/ocaml-packages.nix +++ b/pkgs/top-level/ocaml-packages.nix @@ -321,6 +321,8 @@ let inherit (pkgs) opam git mercurial coreutils gnutar bzip2; }; + dune-site = callPackage ../development/ocaml-modules/dune-site { }; + duration = callPackage ../development/ocaml-modules/duration { }; earley = callPackage ../development/ocaml-modules/earley { }; From a97c084633c004fdc9067a1ae90f16071da2043b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 28 Oct 2021 10:32:32 +0200 Subject: [PATCH 09/32] lnav: 0.10.0 -> 0.10.1 ChangeLog: https://github.com/tstack/lnav/releases/tag/v0.10.1 Closes #140591 --- pkgs/tools/misc/lnav/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/misc/lnav/default.nix b/pkgs/tools/misc/lnav/default.nix index 3cbfb2ad365a..ce541e8f567f 100644 --- a/pkgs/tools/misc/lnav/default.nix +++ b/pkgs/tools/misc/lnav/default.nix @@ -3,13 +3,13 @@ stdenv.mkDerivation rec { pname = "lnav"; - version = "0.10.0"; + version = "0.10.1"; src = fetchFromGitHub { owner = "tstack"; repo = "lnav"; rev = "v${version}"; - sha256 = "sha256-hAFyMypGC065aRgX2L4LdyFGpZ/LnX5SjA2iQv5Iuas="; + sha256 = "sha256-1b4mVKIUotMSK/ADHnpiM42G98JF0abL8sXXGFyS3sw="; }; patches = [ ./0001-Forcefully-disable-docs-build.patch ]; From dded550fcd240bdf3b0acc0740ad174975d3b4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 10:48:34 +0200 Subject: [PATCH 10/32] doc: fix typo --- doc/contributing/coding-conventions.chapter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing/coding-conventions.chapter.md b/doc/contributing/coding-conventions.chapter.md index 7a8e7741a330..79d90c23a40f 100644 --- a/doc/contributing/coding-conventions.chapter.md +++ b/doc/contributing/coding-conventions.chapter.md @@ -181,7 +181,7 @@ rev = "${version}"; ``` -- Filling lists condionally _should_ be done with `lib.optional(s)` instead of using `if cond then [ ... ] else null` or `if cond then [ ... ] else [ ]`. +- Building lists conditionally _should_ be done with `lib.optional(s)` instead of using `if cond then [ ... ] else null` or `if cond then [ ... ] else [ ]`. ```nix buildInputs = lib.optional stdenv.isDarwin iconv; From 252b4731457f1db3e94fa3e1d523b998bd794169 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 28 Oct 2021 09:04:40 +0000 Subject: [PATCH 11/32] terraform-ls: 0.22.0 -> 0.23.0 --- pkgs/development/tools/misc/terraform-ls/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/tools/misc/terraform-ls/default.nix b/pkgs/development/tools/misc/terraform-ls/default.nix index b6eddf743500..8e1f28d5e373 100644 --- a/pkgs/development/tools/misc/terraform-ls/default.nix +++ b/pkgs/development/tools/misc/terraform-ls/default.nix @@ -2,15 +2,15 @@ buildGoModule rec { pname = "terraform-ls"; - version = "0.22.0"; + version = "0.23.0"; src = fetchFromGitHub { owner = "hashicorp"; repo = pname; rev = "v${version}"; - sha256 = "sha256-sfAn9FkOs9/yA7ciRD9gWbx5VwZveqPMYBQhSBkzYlo="; + sha256 = "sha256-hntHEBc1/KSg12x+gv32aNiA2cJ7PWBeftNhI8u+rlw="; }; - vendorSha256 = "sha256-egv2+4esvfYccwmyHm23bec/QN6dGWvJVLG19959LPY="; + vendorSha256 = "sha256-usPIhXWdp/pSAFH0LepG1/Uolfr/1SExinrAjMuB6zg="; ldflags = [ "-s" "-w" "-X main.version=v${version}" "-X main.prerelease=" ]; From 4effa2569a594e1c3b53bdbdc0fae8c3307ba15a Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 28 Oct 2021 09:05:45 +0000 Subject: [PATCH 12/32] python38Packages.phonopy: 2.11.0 -> 2.12.0 --- pkgs/development/python-modules/phonopy/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/phonopy/default.nix b/pkgs/development/python-modules/phonopy/default.nix index 86bece5254ac..dec86d166112 100644 --- a/pkgs/development/python-modules/phonopy/default.nix +++ b/pkgs/development/python-modules/phonopy/default.nix @@ -12,11 +12,11 @@ buildPythonPackage rec { pname = "phonopy"; - version = "2.11.0"; + version = "2.12.0"; src = fetchPypi { inherit pname version; - sha256 = "2ab47d3eb53a5265f3567974d29760b510b3312217358b76efc27ade9bd1a9f0"; + sha256 = "ff65065f418ccbff9fbc1186b9a65581e83b42789aa4a656f45badfff9bd3f61"; }; propagatedBuildInputs = [ From db19e681254e043bffa9e9b47bd64493302ca614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 10:57:01 +0200 Subject: [PATCH 13/32] python39Packages.dogpile-cache: normalise attr, cleanup --- .../python-modules/bugwarrior/default.nix | 4 ++-- .../default.nix | 20 +++++++++---------- .../python-modules/openstacksdk/default.nix | 4 ++-- .../python-ironicclient/default.nix | 4 ++-- .../python-modules/subliminal/default.nix | 4 ++-- pkgs/tools/package-management/nox/default.nix | 2 +- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 2 +- 8 files changed, 20 insertions(+), 21 deletions(-) rename pkgs/development/python-modules/{dogpile.cache => dogpile-cache}/default.nix (70%) diff --git a/pkgs/development/python-modules/bugwarrior/default.nix b/pkgs/development/python-modules/bugwarrior/default.nix index a29e7458d4b5..9beb8e9239ff 100644 --- a/pkgs/development/python-modules/bugwarrior/default.nix +++ b/pkgs/development/python-modules/bugwarrior/default.nix @@ -1,6 +1,6 @@ { lib, buildPythonPackage, fetchPypi, pythonOlder, setuptools , twiggy, requests, offtrac, bugzilla, taskw, python-dateutil, pytz, keyring, six -, jinja2, pycurl, dogpile_cache, lockfile, click, pyxdg, future, jira }: +, jinja2, pycurl, dogpile-cache, lockfile, click, pyxdg, future, jira }: buildPythonPackage rec { pname = "bugwarrior"; @@ -15,7 +15,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ setuptools twiggy requests offtrac bugzilla taskw python-dateutil pytz keyring six - jinja2 pycurl dogpile_cache lockfile click pyxdg future jira + jinja2 pycurl dogpile-cache lockfile click pyxdg future jira ]; # for the moment oauth2client <4.0.0 and megaplan>=1.4 are missing for running the test suite. diff --git a/pkgs/development/python-modules/dogpile.cache/default.nix b/pkgs/development/python-modules/dogpile-cache/default.nix similarity index 70% rename from pkgs/development/python-modules/dogpile.cache/default.nix rename to pkgs/development/python-modules/dogpile-cache/default.nix index 55dffd1dc7e3..5b89c6bdb912 100644 --- a/pkgs/development/python-modules/dogpile.cache/default.nix +++ b/pkgs/development/python-modules/dogpile-cache/default.nix @@ -2,8 +2,7 @@ , buildPythonPackage , fetchPypi , pythonOlder -, pytest -, pytest-cov +, pytestCheckHook , mock , Mako , decorator @@ -11,29 +10,27 @@ }: buildPythonPackage rec { - pname = "dogpile.cache"; + pname = "dogpile-cache"; version = "1.1.4"; disabled = pythonOlder "3.6"; src = fetchPypi { - inherit pname version; + pname = "dogpile.cache"; + inherit version; sha256 = "ea09bebf24bb7c028caf98963785fe9ad0bd397305849a3303bc5380d468d813"; }; - # Disable concurrency tests that often fail, - # probably some kind of timing issue. - postPatch = '' + preCheck = '' + # Disable concurrency tests that often fail, + # probably some kind of timing issue. rm tests/test_lock.py # Failing tests. https://bitbucket.org/zzzeek/dogpile.cache/issues/116 rm tests/cache/test_memcached_backend.py ''; dontUseSetuptoolsCheck = true; - checkPhase = '' - pytest - ''; - checkInputs = [ pytest pytest-cov mock Mako ]; + checkInputs = [ pytestCheckHook mock Mako ]; propagatedBuildInputs = [ decorator stevedore ]; @@ -41,5 +38,6 @@ buildPythonPackage rec { description = "A caching front-end based on the Dogpile lock"; homepage = "https://bitbucket.org/zzzeek/dogpile.cache"; license = licenses.bsd3; + maintainers = with maintainers; [ ]; }; } diff --git a/pkgs/development/python-modules/openstacksdk/default.nix b/pkgs/development/python-modules/openstacksdk/default.nix index ec45fd8d09b8..27940dde7670 100644 --- a/pkgs/development/python-modules/openstacksdk/default.nix +++ b/pkgs/development/python-modules/openstacksdk/default.nix @@ -4,7 +4,7 @@ , fetchPypi , appdirs , cryptography -, dogpile_cache +, dogpile-cache , jmespath , jsonpatch , keystoneauth1 @@ -29,7 +29,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ appdirs cryptography - dogpile_cache + dogpile-cache jmespath jsonpatch keystoneauth1 diff --git a/pkgs/development/python-modules/python-ironicclient/default.nix b/pkgs/development/python-modules/python-ironicclient/default.nix index fb4840753df6..5654c2de3a85 100644 --- a/pkgs/development/python-modules/python-ironicclient/default.nix +++ b/pkgs/development/python-modules/python-ironicclient/default.nix @@ -4,7 +4,7 @@ , pbr , appdirs , cliff -, dogpile_cache +, dogpile-cache , jsonschema , keystoneauth1 , openstacksdk @@ -31,7 +31,7 @@ buildPythonApplication rec { pbr appdirs cliff - dogpile_cache + dogpile-cache jsonschema keystoneauth1 openstacksdk diff --git a/pkgs/development/python-modules/subliminal/default.nix b/pkgs/development/python-modules/subliminal/default.nix index e334608bd4dc..9188ad223eaa 100644 --- a/pkgs/development/python-modules/subliminal/default.nix +++ b/pkgs/development/python-modules/subliminal/default.nix @@ -7,7 +7,7 @@ , beautifulsoup4 , requests , click -, dogpile_cache +, dogpile-cache , stevedore , chardet , pysrt @@ -34,7 +34,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ guessit babelfish enzyme beautifulsoup4 requests - click dogpile_cache stevedore chardet pysrt six + click dogpile-cache stevedore chardet pysrt six appdirs rarfile pytz ]; diff --git a/pkgs/tools/package-management/nox/default.nix b/pkgs/tools/package-management/nox/default.nix index bf98f5400b8e..c949b443e363 100644 --- a/pkgs/tools/package-management/nox/default.nix +++ b/pkgs/tools/package-management/nox/default.nix @@ -15,7 +15,7 @@ python3Packages.buildPythonApplication rec { buildInputs = [ python3Packages.pbr git ]; propagatedBuildInputs = with python3Packages; [ - dogpile_cache + dogpile-cache click requests characteristic diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index f53f4543319a..e8aac50d7636 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -45,6 +45,7 @@ mapAliases ({ discogs_client = discogs-client; # added 2021-07-02 djangorestframework-jwt = drf-jwt; dns = dnspython; # Alias for compatibility, 2017-12-10 + dogpile_cache = dogpile-cache; # added 2021-10-28 faulthandler = throw "faulthandler is built into ${python.executable}"; gitdb2 = throw "gitdb2 has been deprecated, use gitdb instead."; # added 2020-03-14 glances = throw "glances has moved to pkgs.glances"; # added 2020-20-28 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 02c59fa0fb63..39f9151b8ee2 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2255,7 +2255,7 @@ in { dodgy = callPackage ../development/python-modules/dodgy { }; - dogpile_cache = callPackage ../development/python-modules/dogpile.cache { }; + dogpile-cache = callPackage ../development/python-modules/dogpile-cache { }; dogpile_core = callPackage ../development/python-modules/dogpile.core { }; From 846ceff3b324f25e6bb202f6e1ea33194a0d2fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 10:59:03 +0200 Subject: [PATCH 14/32] python39Packages.sqlalchemy-migrate: normalise attr --- pkgs/applications/misc/openlp/lib.nix | 4 ++-- pkgs/applications/misc/pytrainer/default.nix | 2 +- pkgs/development/python-modules/buildbot/default.nix | 4 ++-- pkgs/development/python-modules/oslo-db/default.nix | 4 ++-- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkgs/applications/misc/openlp/lib.nix b/pkgs/applications/misc/openlp/lib.nix index c044e76c597d..9e8440a4c806 100644 --- a/pkgs/applications/misc/openlp/lib.nix +++ b/pkgs/applications/misc/openlp/lib.nix @@ -5,7 +5,7 @@ # python deps , python, buildPythonPackage , alembic, beautifulsoup4, chardet, lxml, Mako, pyenchant -, pyqt5_with_qtwebkit, pyxdg, sip_4, sqlalchemy, sqlalchemy_migrate +, pyqt5_with_qtwebkit, pyxdg, sip_4, sqlalchemy, sqlalchemy-migrate }: buildPythonPackage rec { @@ -43,7 +43,7 @@ buildPythonPackage rec { pyxdg sip_4 sqlalchemy - sqlalchemy_migrate + sqlalchemy-migrate ]; prePatch = '' diff --git a/pkgs/applications/misc/pytrainer/default.nix b/pkgs/applications/misc/pytrainer/default.nix index 05cc834591f5..d7969155370e 100644 --- a/pkgs/applications/misc/pytrainer/default.nix +++ b/pkgs/applications/misc/pytrainer/default.nix @@ -36,7 +36,7 @@ in python.pkgs.buildPythonApplication rec { }; propagatedBuildInputs = with python.pkgs; [ - sqlalchemy_migrate + sqlalchemy-migrate python-dateutil matplotlib lxml diff --git a/pkgs/development/python-modules/buildbot/default.nix b/pkgs/development/python-modules/buildbot/default.nix index 3c5db01cdb68..79aaf4e08559 100644 --- a/pkgs/development/python-modules/buildbot/default.nix +++ b/pkgs/development/python-modules/buildbot/default.nix @@ -1,6 +1,6 @@ { stdenv, lib, buildPythonPackage, fetchPypi, makeWrapper, isPy3k , python, twisted, jinja2, zope_interface, sqlalchemy -, sqlalchemy_migrate, python-dateutil, txaio, autobahn, pyjwt, pyyaml, unidiff, treq +, sqlalchemy-migrate, python-dateutil, txaio, autobahn, pyjwt, pyyaml, unidiff, treq , txrequests, pypugjs, boto3, moto, mock, lz4, setuptoolsTrial , isort, pylint, flake8, buildbot-worker, buildbot-pkg, buildbot-plugins , parameterized, git, openssh, glibcLocales, ldap3, nixosTests @@ -44,7 +44,7 @@ let jinja2 zope_interface sqlalchemy - sqlalchemy_migrate + sqlalchemy-migrate python-dateutil txaio autobahn diff --git a/pkgs/development/python-modules/oslo-db/default.nix b/pkgs/development/python-modules/oslo-db/default.nix index c0c9dff04334..f0f2e1f0ee72 100644 --- a/pkgs/development/python-modules/oslo-db/default.nix +++ b/pkgs/development/python-modules/oslo-db/default.nix @@ -8,7 +8,7 @@ , oslotest , pbr , sqlalchemy -, sqlalchemy_migrate +, sqlalchemy-migrate , stestr , testresources , testscenarios @@ -32,7 +32,7 @@ buildPythonPackage rec { oslo-context oslo-utils sqlalchemy - sqlalchemy_migrate + sqlalchemy-migrate testresources testscenarios ]; diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index e8aac50d7636..355bcbb6e2b2 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -83,6 +83,7 @@ mapAliases ({ smart_open = smart-open; # added 2021-03-14 smmap2 = throw "smmap2 has been deprecated, use smmap instead."; # added 2020-03-14 sphinxcontrib_plantuml = sphinxcontrib-plantuml; + sqlalchemy_migrate = sqlalchemy-migrate; # added 2021-10-28 topydo = throw "python3Packages.topydo was moved to topydo"; # 2017-09-22 tvnamer = throw "python3Packages.tvnamer was moved to tvnamer"; # 2021-07-05 WazeRouteCalculator = wazeroutecalculator; # 2021-09-29 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 39f9151b8ee2..5ad54e7893d1 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -8778,7 +8778,7 @@ in { sqlalchemy-jsonfield = callPackage ../development/python-modules/sqlalchemy-jsonfield { }; - sqlalchemy_migrate = callPackage ../development/python-modules/sqlalchemy-migrate { }; + sqlalchemy-migrate = callPackage ../development/python-modules/sqlalchemy-migrate { }; sqlalchemy-utils = callPackage ../development/python-modules/sqlalchemy-utils { }; From 2b218f52c8edc554c8ad492300e4e6ad2c8b897e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 11:03:31 +0200 Subject: [PATCH 15/32] python39Packages: normalise aliases and add missing comments --- pkgs/top-level/python-aliases.nix | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 355bcbb6e2b2..c6dca5addf99 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -33,20 +33,20 @@ in ### Deprecated aliases - for backward compatibility mapAliases ({ - blockdiagcontrib-cisco = throw "blockdiagcontrib-cisco is not compatible with blockdiag 2.0.0 and has been removed."; # Added 2020-11-29 + blockdiagcontrib-cisco = throw "blockdiagcontrib-cisco is not compatible with blockdiag 2.0.0 and has been removed."; # added 2020-11-29 bt_proximity = bt-proximity; # added 2021-07-02 - bugseverywhere = throw "bugseverywhere has been removed: Abandoned by upstream."; # Added 2019-11-27 + bugseverywhere = throw "bugseverywhere has been removed: Abandoned by upstream."; # added 2019-11-27 class-registry = phx-class-registry; # added 2021-10-05 ConfigArgParse = configargparse; # added 2021-03-18 dateutil = python-dateutil; # added 2021-07-03 detox = throw "detox is no longer maintained, and was broken since may 2019"; # added 2020-07-04 - dftfit = throw "it's dependency lammps-cython no longer builds"; + dftfit = throw "it's dependency lammps-cython no longer builds"; # added 2021-07-04 diff_cover = diff-cover; # added 2021-07-02 discogs_client = discogs-client; # added 2021-07-02 - djangorestframework-jwt = drf-jwt; - dns = dnspython; # Alias for compatibility, 2017-12-10 + djangorestframework-jwt = drf-jwt; # added 2021-07-20 + dns = dnspython; # added 2017-12-10 dogpile_cache = dogpile-cache; # added 2021-10-28 - faulthandler = throw "faulthandler is built into ${python.executable}"; + faulthandler = throw "faulthandler is built into ${python.executable}"; # added 2021-07-12 gitdb2 = throw "gitdb2 has been deprecated, use gitdb instead."; # added 2020-03-14 glances = throw "glances has moved to pkgs.glances"; # added 2020-20-28 google_api_python_client = google-api-python-client; # added 2021-03-19 @@ -57,11 +57,11 @@ mapAliases ({ lammps-cython = throw "no longer builds and is unmaintained"; MechanicalSoup = mechanicalsoup; # added 2021-06-01 pam = python-pam; # added 2020-09-07. - PasteDeploy = pastedeploy; - powerlineMemSegment = powerline-mem-segment; + PasteDeploy = pastedeploy; # added 2021-10-07 + powerlineMemSegment = powerline-mem-segment; # added 2021-10-08 privacyidea = throw "renamed to pkgs.privacyidea"; # added 2021-06-20 prometheus_client = prometheus-client; # added 2021-06-10 - prompt_toolkit = prompt-toolkit; + prompt_toolkit = prompt-toolkit; # added 2021-07-22 pylibgen = throw "pylibgen is unmaintained upstreamed, and removed from nixpkgs"; # added 2020-06-20 pymssql = throw "pymssql has been abandoned upstream."; # added 2020-05-04 pysmart-smartx = pysmart; # added 2021-10-22 @@ -78,15 +78,15 @@ mapAliases ({ requests_toolbelt = requests-toolbelt; # added 2017-09-26 rotate-backups = throw "pythonPackages.rotate-backups was removed in favor of the top-level rotate-backups"; # added 2021-07-01 scikitlearn = scikit-learn; # added 2021-07-21 - selectors34 = throw "selectors34 has been removed: functionality provided by Python itself; archived by upstream."; # Added 2021-06-10 + selectors34 = throw "selectors34 has been removed: functionality provided by Python itself; archived by upstream."; # added 2021-06-10 setuptools_scm = setuptools-scm; # added 2021-06-03 smart_open = smart-open; # added 2021-03-14 smmap2 = throw "smmap2 has been deprecated, use smmap instead."; # added 2020-03-14 - sphinxcontrib_plantuml = sphinxcontrib-plantuml; + sphinxcontrib_plantuml = sphinxcontrib-plantuml; # added 2021-08-02 sqlalchemy_migrate = sqlalchemy-migrate; # added 2021-10-28 - topydo = throw "python3Packages.topydo was moved to topydo"; # 2017-09-22 - tvnamer = throw "python3Packages.tvnamer was moved to tvnamer"; # 2021-07-05 - WazeRouteCalculator = wazeroutecalculator; # 2021-09-29 - websocket_client = websocket-client; + topydo = throw "python3Packages.topydo was moved to topydo"; # added 2017-09-22 + tvnamer = throw "python3Packages.tvnamer was moved to tvnamer"; # added 2021-07-05 + WazeRouteCalculator = wazeroutecalculator; # added 2021-09-29 + websocket_client = websocket-client; # added 2021-06-15 zc-buildout221 = zc-buildout; # added 2021-07-21 }) From 71ed943a867ba602eec6feabd9ada703993df87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 11:05:19 +0200 Subject: [PATCH 16/32] python39Packages.imapclient: normalise name --- pkgs/development/python-modules/deltachat/default.nix | 4 ++-- pkgs/development/python-modules/imapclient/default.nix | 2 +- pkgs/development/python-modules/mailsuite/default.nix | 4 ++-- pkgs/development/python-modules/parsedmarc/default.nix | 4 ++-- pkgs/top-level/python-aliases.nix | 1 + pkgs/top-level/python-packages.nix | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/deltachat/default.nix b/pkgs/development/python-modules/deltachat/default.nix index ffa9b8d7d63a..78f42bdfceb0 100644 --- a/pkgs/development/python-modules/deltachat/default.nix +++ b/pkgs/development/python-modules/deltachat/default.nix @@ -5,7 +5,7 @@ , setuptools-scm , libdeltachat , cffi -, IMAPClient +, imapclient , pluggy , requests , setuptools @@ -31,7 +31,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ cffi - IMAPClient + imapclient pluggy requests setuptools diff --git a/pkgs/development/python-modules/imapclient/default.nix b/pkgs/development/python-modules/imapclient/default.nix index ea4f388b6c7e..c1394a7b6b53 100644 --- a/pkgs/development/python-modules/imapclient/default.nix +++ b/pkgs/development/python-modules/imapclient/default.nix @@ -7,7 +7,7 @@ }: buildPythonPackage rec { - pname = "IMAPClient"; + pname = "imapclient"; version = "2.2.0"; src = fetchFromGitHub { diff --git a/pkgs/development/python-modules/mailsuite/default.nix b/pkgs/development/python-modules/mailsuite/default.nix index 23167af45b0e..8d1046387ef0 100644 --- a/pkgs/development/python-modules/mailsuite/default.nix +++ b/pkgs/development/python-modules/mailsuite/default.nix @@ -7,7 +7,7 @@ , dnspython , html2text , mail-parser -, IMAPClient +, imapclient }: buildPythonPackage rec { @@ -25,7 +25,7 @@ buildPythonPackage rec { dnspython html2text mail-parser - IMAPClient + imapclient ]; pythonImportsCheck = [ "mailsuite" ]; diff --git a/pkgs/development/python-modules/parsedmarc/default.nix b/pkgs/development/python-modules/parsedmarc/default.nix index b9fb35016c3b..7cba1c227a85 100644 --- a/pkgs/development/python-modules/parsedmarc/default.nix +++ b/pkgs/development/python-modules/parsedmarc/default.nix @@ -14,7 +14,7 @@ , publicsuffix2 , xmltodict , geoip2 -, IMAPClient +, imapclient , dateparser , elasticsearch-dsl , kafka-python @@ -49,7 +49,7 @@ buildPythonPackage rec { publicsuffix2 xmltodict geoip2 - IMAPClient + imapclient dateparser elasticsearch-dsl kafka-python diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index c6dca5addf99..43d8c3ef8a95 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -53,6 +53,7 @@ mapAliases ({ googleapis_common_protos = googleapis-common-protos; # added 2021-03-19 grpc_google_iam_v1 = grpc-google-iam-v1; # added 2021-08-21 HAP-python = hap-python; # added 2021-06-01 + IMAPClient = imapclient; # added 2021-10-28 jupyter_client = jupyter-client; # added 2021-10-15 lammps-cython = throw "no longer builds and is unmaintained"; MechanicalSoup = mechanicalsoup; # added 2021-06-01 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5ad54e7893d1..b25cc0339c4a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3666,7 +3666,7 @@ in { imantics = callPackage ../development/python-modules/imantics { }; - IMAPClient = callPackage ../development/python-modules/imapclient { }; + imapclient = callPackage ../development/python-modules/imapclient { }; imaplib2 = callPackage ../development/python-modules/imaplib2 { }; From a0dba13edeb10264ba664ea282d91f27f91ed033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 28 Oct 2021 11:11:08 +0200 Subject: [PATCH 17/32] python39Packages.dogpile-core: normalise name --- .../{dogpile.core => dogpile-core}/default.nix | 8 ++++++-- pkgs/top-level/python-packages.nix | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) rename pkgs/development/python-modules/{dogpile.core => dogpile-core}/default.nix (72%) diff --git a/pkgs/development/python-modules/dogpile.core/default.nix b/pkgs/development/python-modules/dogpile-core/default.nix similarity index 72% rename from pkgs/development/python-modules/dogpile.core/default.nix rename to pkgs/development/python-modules/dogpile-core/default.nix index 8ef37282250d..32a1f59e9270 100644 --- a/pkgs/development/python-modules/dogpile.core/default.nix +++ b/pkgs/development/python-modules/dogpile-core/default.nix @@ -1,19 +1,23 @@ { lib, buildPythonPackage, fetchPypi }: buildPythonPackage rec { - pname = "dogpile.core"; + pname = "dogpile-core"; version = "0.4.1"; src = fetchPypi { - inherit pname version; + pname = "dogpile.core"; + inherit version; sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy"; }; doCheck = false; + pythonImportsCheck = [ "dogpile.core" ]; + meta = with lib; { description = "A 'dogpile' lock, typically used as a component of a larger caching solution"; homepage = "https://bitbucket.org/zzzeek/dogpile.core"; license = licenses.bsd3; + maintainers = with maintainers; [ ]; }; } diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b25cc0339c4a..22ad1d220a72 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2257,7 +2257,7 @@ in { dogpile-cache = callPackage ../development/python-modules/dogpile-cache { }; - dogpile_core = callPackage ../development/python-modules/dogpile.core { }; + dogpile-core = callPackage ../development/python-modules/dogpile-core { }; dogtail = callPackage ../development/python-modules/dogtail { }; From 9ce555e0afaefa5715c6d89067c81b7418e91640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Thu, 14 Oct 2021 21:47:43 -0700 Subject: [PATCH 18/32] newsboat: 2.24 -> 2.25 https://github.com/newsboat/newsboat/releases/tag/r2.25 --- .../networking/feedreaders/newsboat/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/networking/feedreaders/newsboat/default.nix b/pkgs/applications/networking/feedreaders/newsboat/default.nix index 289f631ee8fb..65835aa10cf3 100644 --- a/pkgs/applications/networking/feedreaders/newsboat/default.nix +++ b/pkgs/applications/networking/feedreaders/newsboat/default.nix @@ -3,16 +3,16 @@ rustPlatform.buildRustPackage rec { pname = "newsboat"; - version = "2.24"; + version = "2.25"; src = fetchFromGitHub { owner = "newsboat"; repo = "newsboat"; rev = "r${version}"; - sha256 = "0qb10w7pn9hiczjjpc0xanmc83p22ngcqcrkjwji5rl0sh8v9ii4"; + sha256 = "sha256-TAnGDxTKYl4niouS6nYdJDaIngAPsPHr9Bw9L3cR2Xk="; }; - cargoSha256 = "0vhr8qyw4f3lc208mgl1m3z42igkbwly6wcv7g7jrq3c6dcwyhjn"; + cargoSha256 = "sha256-MxoyYBLbrCuLVa0p8JrYKSKu2oFPnXMwab42lhhAu48="; # TODO: Check if that's still needed postPatch = lib.optionalString stdenv.isDarwin '' From cb788cef507014364430a0ef7ead842879c95762 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 28 Oct 2021 11:35:58 +0200 Subject: [PATCH 19/32] gitea: 1.15.5 -> 1.15.6 ChangeLog: https://github.com/go-gitea/gitea/releases/tag/v1.15.6 --- pkgs/applications/version-management/gitea/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/version-management/gitea/default.nix b/pkgs/applications/version-management/gitea/default.nix index de0e97501fd4..ab84a5143d28 100644 --- a/pkgs/applications/version-management/gitea/default.nix +++ b/pkgs/applications/version-management/gitea/default.nix @@ -16,12 +16,12 @@ with lib; buildGoPackage rec { pname = "gitea"; - version = "1.15.5"; + version = "1.15.6"; # not fetching directly from the git repo, because that lacks several vendor files for the web UI src = fetchurl { url = "https://github.com/go-gitea/gitea/releases/download/v${version}/gitea-src-${version}.tar.gz"; - sha256 = "sha256-W+czWzo4keTLRPDLcTeYl3JSccwfq+P6k4ELADO9FVM="; + sha256 = "sha256-FMM/iQAxJcymv4jYBzaBXG0Uy8UxHh9gFWB5gzV9cn0="; }; unpackPhase = '' From 89707c93aa20187153b90dc8868b595e79236a12 Mon Sep 17 00:00:00 2001 From: Nikolay Korotkiy Date: Thu, 28 Oct 2021 11:49:34 +0300 Subject: [PATCH 20/32] pcalc: enable on darwin --- pkgs/applications/science/math/pcalc/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/science/math/pcalc/default.nix b/pkgs/applications/science/math/pcalc/default.nix index cbf07135dec2..97888b712eb1 100644 --- a/pkgs/applications/science/math/pcalc/default.nix +++ b/pkgs/applications/science/math/pcalc/default.nix @@ -19,8 +19,8 @@ stdenv.mkDerivation rec { meta = with lib; { homepage = "https://vapier.github.io/pcalc/"; description = "Programmer's calculator"; - license = licenses.gpl2; - maintainers = with lib.maintainers; [ ftrvxmtrx ]; - platforms = lib.platforms.linux; + license = licenses.gpl2Plus; + maintainers = with maintainers; [ ftrvxmtrx ]; + platforms = platforms.unix; }; } From b7c19f700d3df0e727382d720810700b7cd8c8b8 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 28 Oct 2021 10:53:42 +0000 Subject: [PATCH 21/32] tz: 0.5 -> 0.6.1 --- pkgs/tools/misc/tz/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/misc/tz/default.nix b/pkgs/tools/misc/tz/default.nix index c46137bd9e5a..8148dfdec702 100644 --- a/pkgs/tools/misc/tz/default.nix +++ b/pkgs/tools/misc/tz/default.nix @@ -2,16 +2,16 @@ buildGoModule rec { pname = "tz"; - version = "0.5"; + version = "0.6.1"; src = fetchFromGitHub { owner = "oz"; repo = "tz"; rev = "v${version}"; - sha256 = "sha256-OwjhV3n1B1yQTNYm4VOW500t0524g85YYiOAAu9yPeo="; + sha256 = "sha256-D0rakLZ+swrDwBMcr+EJPgaYsQTWob50QteW1PoIdNk="; }; - vendorSha256 = "sha256-Soa87I7oMa34LjYKxNAz9Limi0kQ6JUtb/zI4G7yZnw="; + vendorSha256 = "sha256-gMHPWf/kFb55GyyMciLWRzvrQXUGDQ72RNkIhkMVe54="; meta = with lib; { description = "A time zone helper"; From 56c5efa25b51e4a1b8452ef2adb18d96dadfb323 Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 5 Oct 2021 15:50:04 +0200 Subject: [PATCH 22/32] make-disk-image: Reintroduce the installBootLoader argument installBootLoader was removed in f1708a9d7d79e2bf2961fc648625578b23b3460f; probably by mistake, since only the logic was removed, not the argument. --- nixos/lib/make-disk-image.nix | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix index 55643facea03..63dd5594af30 100644 --- a/nixos/lib/make-disk-image.nix +++ b/nixos/lib/make-disk-image.nix @@ -44,6 +44,9 @@ # most likely fails as GRUB will probably refuse to install. partitionTableType ? "legacy" +, # Whether to invoke `switch-to-configuration boot` during image creation + installBootLoader ? true + , # The root file system type. fsType ? "ext4" @@ -368,11 +371,13 @@ in pkgs.vmTools.runInLinuxVM ( cp ${configFile} /mnt/etc/nixos/configuration.nix ''} - # Set up core system link, GRUB, etc. - NIXOS_INSTALL_BOOTLOADER=1 nixos-enter --root $mountPoint -- /nix/var/nix/profiles/system/bin/switch-to-configuration boot + ${lib.optionalString installBootLoader '' + # Set up core system link, GRUB, etc. + NIXOS_INSTALL_BOOTLOADER=1 nixos-enter --root $mountPoint -- /nix/var/nix/profiles/system/bin/switch-to-configuration boot - # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images - rm -f $mountPoint/etc/machine-id + # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images + rm -f $mountPoint/etc/machine-id + ''} # Set the ownerships of the contents. The modes are set in preVM. # No globbing on targets, so no need to set -f From 398a73ac980f2e89981d8d704d7e800bb7a9bfaf Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 5 Oct 2021 19:08:57 +0200 Subject: [PATCH 23/32] make-disk-image: Add copyChannel argument Add a copyChannel argument which controls whether the current source tree will be made available as a nix channel in the image or not. Previously, it always was. Making it available is useful for interactive use of nix utils, but changes the hash of the image when the sources are updated. --- nixos/lib/make-disk-image.nix | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix index 63dd5594af30..d302fae430c1 100644 --- a/nixos/lib/make-disk-image.nix +++ b/nixos/lib/make-disk-image.nix @@ -64,6 +64,12 @@ , # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw. format ? "raw" + +, # Whether a nix channel based on the current source tree should be + # made available inside the image. Useful for interactive use of nix + # utils, but changes the hash of the image when the sources are + # updated. + copyChannel ? true }: assert partitionTableType == "legacy" || partitionTableType == "legacy+gpt" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none"; @@ -166,7 +172,9 @@ let format' = format; in let users = map (x: x.user or "''") contents; groups = map (x: x.group or "''") contents; - closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; }; + closureInfo = pkgs.closureInfo { + rootPaths = [ config.system.build.toplevel ] ++ (lib.optional copyChannel channelSources); + }; blockSize = toString (4 * 1024); # ext4fs block size (not block device sector size) @@ -254,7 +262,9 @@ let format' = format; in let chmod 755 "$TMPDIR" echo "running nixos-install..." nixos-install --root $root --no-bootloader --no-root-passwd \ - --system ${config.system.build.toplevel} --channel ${channelSources} --substituters "" + --system ${config.system.build.toplevel} \ + ${if copyChannel then "--channel ${channelSources}" else "--no-channel-copy"} \ + --substituters "" diskImage=nixos.raw From 87d8eec0691caf16b924ad7401a29e2783b12896 Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 5 Oct 2021 20:56:25 +0200 Subject: [PATCH 24/32] make-disk-image: Add additionalPaths argument Add an argument which makes it possible to add additional paths to the image's nix store. --- nixos/lib/make-disk-image.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix index d302fae430c1..9bbfc6c490e3 100644 --- a/nixos/lib/make-disk-image.nix +++ b/nixos/lib/make-disk-image.nix @@ -70,6 +70,9 @@ # utils, but changes the hash of the image when the sources are # updated. copyChannel ? true + +, # Additional store paths to copy to the image's store. + additionalPaths ? [] }: assert partitionTableType == "legacy" || partitionTableType == "legacy+gpt" || partitionTableType == "efi" || partitionTableType == "hybrid" || partitionTableType == "none"; @@ -172,8 +175,13 @@ let format' = format; in let users = map (x: x.user or "''") contents; groups = map (x: x.group or "''") contents; + basePaths = [ config.system.build.toplevel ] + ++ lib.optional copyChannel channelSources; + + additionalPaths' = subtractLists basePaths additionalPaths; + closureInfo = pkgs.closureInfo { - rootPaths = [ config.system.build.toplevel ] ++ (lib.optional copyChannel channelSources); + rootPaths = basePaths ++ additionalPaths'; }; blockSize = toString (4 * 1024); # ext4fs block size (not block device sector size) @@ -266,6 +274,10 @@ let format' = format; in let ${if copyChannel then "--channel ${channelSources}" else "--no-channel-copy"} \ --substituters "" + ${optionalString (additionalPaths' != []) '' + nix copy --to $root --no-check-sigs ${concatStringsSep " " additionalPaths'} + ''} + diskImage=nixos.raw ${if diskSize == "auto" then '' From 84839b395f5740789af7a843aaeea80ac2f6f60b Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 5 Oct 2021 22:32:56 +0200 Subject: [PATCH 25/32] nixos/qemu-vm: Allow building a Nix store image instead of using 9p Add the `useNixStoreImage` option, allowing a disk image with the necessary contents from the Nix store to be built using make-disk-image.nix. The image will be mounted at `/nix/store` and acts as a drop-in replacement for the usual 9p mounting of the host's Nix store. This removes the performance penalty of 9p, drastically improving execution speed of applications which do lots of reads from the Nix store. The caveats are increased disk space usage and image build time. --- nixos/modules/virtualisation/qemu-vm.nix | 59 ++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index c5f71b249a6d..6af5848b1682 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -122,6 +122,12 @@ let TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir) fi + ${lib.optionalString cfg.useNixStoreImage + '' + # Create a writable copy/snapshot of the store image. + ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img + ''} + # Create a directory for exchanging data with the VM. mkdir -p "$TMPDIR/xchg" @@ -263,6 +269,18 @@ let '' # */ ); + storeImage = import ../../lib/make-disk-image.nix { + inherit pkgs config lib; + additionalPaths = [ regInfo ]; + format = "qcow2"; + onlyNixStore = true; + partitionTableType = "none"; + installBootLoader = false; + diskSize = "auto"; + additionalSpace = "0M"; + copyChannel = false; + }; + in { @@ -608,6 +626,20 @@ in }; }; + virtualisation.useNixStoreImage = + mkOption { + type = types.bool; + default = false; + description = '' + Build and use a disk image for the Nix store, instead of + accessing the host's one through 9p. + + For applications which do a lot of reads from the store, + this can drastically improve performance, but at the cost of + disk space and image build time. + ''; + }; + virtualisation.useBootLoader = mkOption { type = types.bool; @@ -762,9 +794,18 @@ in virtualisation.pathsInNixDB = [ config.system.build.toplevel ]; virtualisation.sharedDirectories = { - nix-store = { source = "/nix/store"; target = "/nix/store"; }; - xchg = { source = ''"$TMPDIR"/xchg''; target = "/tmp/xchg"; }; - shared = { source = ''"''${SHARED_DIR:-$TMPDIR/xchg}"''; target = "/tmp/shared"; }; + nix-store = mkIf (!cfg.useNixStoreImage) { + source = builtins.storeDir; + target = "/nix/store"; + }; + xchg = { + source = ''"$TMPDIR"/xchg''; + target = "/tmp/xchg"; + }; + shared = { + source = ''"''${SHARED_DIR:-$TMPDIR/xchg}"''; + target = "/tmp/shared"; + }; }; virtualisation.qemu.networkingOptions = @@ -815,6 +856,11 @@ in driveExtraOpts.cache = "writeback"; driveExtraOpts.werror = "report"; }] + (mkIf cfg.useNixStoreImage [{ + name = "nix-store"; + file = ''"$TMPDIR"/store.img''; + deviceExtraOpts.bootindex = if cfg.useBootLoader then "3" else "2"; + }]) (mkIf cfg.useBootLoader [ # The order of this list determines the device names, see # note [Disk layout with `useBootLoader`]. @@ -865,6 +911,13 @@ in options = [ "mode=1777" "strictatime" "nosuid" "nodev" "size=${toString config.boot.tmpOnTmpfsSize}" ]; }; + "/nix/${if cfg.writableStore then ".ro-store" else "store"}" = + mkIf cfg.useNixStoreImage + { device = "${lookupDriveDeviceName "nix-store" cfg.qemu.drives}"; + neededForBoot = true; + options = [ "ro" ]; + }; + "/nix/.rw-store" = mkIf (cfg.writableStore && cfg.writableStoreUseTmpfs) { fsType = "tmpfs"; options = [ "mode=0755" ]; From af9f6d9a2aeb279408bcbb90db16ef3f727e2601 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 6 Oct 2021 19:19:29 +0200 Subject: [PATCH 26/32] nixos/qemu-vm: Rename pathsInNixDB to additionalPaths pathsInNixDB isn't a very accurate name when a Nix store image is built (virtualisation.useNixStoreImage); rename it to additionalPaths, which should be general enough to cover both cases. --- nixos/modules/virtualisation/qemu-vm.nix | 21 ++++++++++++++------- nixos/tests/containers-bridge.nix | 2 +- nixos/tests/containers-extra_veth.nix | 2 +- nixos/tests/containers-imperative.nix | 2 +- nixos/tests/containers-ip.nix | 2 +- nixos/tests/containers-portforward.nix | 2 +- nixos/tests/containers-tmpfs.nix | 2 +- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 6af5848b1682..b7a03eaac751 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -177,7 +177,7 @@ let ''; - regInfo = pkgs.closureInfo { rootPaths = config.virtualisation.pathsInNixDB; }; + regInfo = pkgs.closureInfo { rootPaths = config.virtualisation.additionalPaths; }; # Generate a hard disk image containing a /boot partition and GRUB @@ -286,6 +286,7 @@ in { imports = [ ../profiles/qemu-guest.nix + (mkRenamedOptionModule [ "virtualisation" "pathsInNixDB" ] [ "virtualisation" "additionalPaths" ]) ]; options = { @@ -417,17 +418,23 @@ in ''; }; - virtualisation.pathsInNixDB = + virtualisation.additionalPaths = mkOption { type = types.listOf types.path; default = []; description = '' - The list of paths whose closure is registered in the Nix - database in the VM. All other paths in the host Nix store + A list of paths whose closure should be made available to + the VM. + + When 9p is used, the closure is registered in the Nix + database in the VM. All other paths in the host Nix store appear in the guest Nix store as well, but are considered garbage (because they are not registered in the Nix - database in the guest). + database of the guest). + + When is + set, the closure is copied to the Nix store image. ''; }; @@ -772,7 +779,7 @@ in ''; # After booting, register the closure of the paths in - # `virtualisation.pathsInNixDB' in the Nix database in the VM. This + # `virtualisation.additionalPaths' in the Nix database in the VM. This # allows Nix operations to work in the VM. The path to the # registration file is passed through the kernel command line to # allow `system.build.toplevel' to be included. (If we had a direct @@ -791,7 +798,7 @@ in virtualisation.bootDevice = mkDefault (driveDeviceName 1); - virtualisation.pathsInNixDB = [ config.system.build.toplevel ]; + virtualisation.additionalPaths = [ config.system.build.toplevel ]; virtualisation.sharedDirectories = { nix-store = mkIf (!cfg.useNixStoreImage) { diff --git a/nixos/tests/containers-bridge.nix b/nixos/tests/containers-bridge.nix index 12fa67c8b015..a1b0c9469d83 100644 --- a/nixos/tests/containers-bridge.nix +++ b/nixos/tests/containers-bridge.nix @@ -56,7 +56,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { }; - virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv ]; }; testScript = '' diff --git a/nixos/tests/containers-extra_veth.nix b/nixos/tests/containers-extra_veth.nix index cbbb25258325..172409f56e89 100644 --- a/nixos/tests/containers-extra_veth.nix +++ b/nixos/tests/containers-extra_veth.nix @@ -45,7 +45,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { }; }; - virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv ]; }; testScript = diff --git a/nixos/tests/containers-imperative.nix b/nixos/tests/containers-imperative.nix index 1dcccfc306a3..361160b1fdc2 100644 --- a/nixos/tests/containers-imperative.nix +++ b/nixos/tests/containers-imperative.nix @@ -17,7 +17,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { virtualisation.memorySize = 1024; # Make sure we always have all the required dependencies for creating a # container available within the VM, because we don't have network access. - virtualisation.pathsInNixDB = let + virtualisation.additionalPaths = let emptyContainer = import ../lib/eval-config.nix { inherit (config.nixpkgs.localSystem) system; modules = lib.singleton { diff --git a/nixos/tests/containers-ip.nix b/nixos/tests/containers-ip.nix index 5abea2dbad9f..8fc42dab6241 100644 --- a/nixos/tests/containers-ip.nix +++ b/nixos/tests/containers-ip.nix @@ -27,7 +27,7 @@ in import ./make-test-python.nix ({ pkgs, lib, ... }: { containers.webserver4 = webserverFor "10.231.136.1" "10.231.136.2"; containers.webserver6 = webserverFor "fc00::2" "fc00::1"; - virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv ]; }; testScript = { nodes, ... }: '' diff --git a/nixos/tests/containers-portforward.nix b/nixos/tests/containers-portforward.nix index 221a6f50efd1..e21f6cee76a9 100644 --- a/nixos/tests/containers-portforward.nix +++ b/nixos/tests/containers-portforward.nix @@ -29,7 +29,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { }; }; - virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv ]; }; testScript = diff --git a/nixos/tests/containers-tmpfs.nix b/nixos/tests/containers-tmpfs.nix index fd9f9a252ca8..0185c2d91f23 100644 --- a/nixos/tests/containers-tmpfs.nix +++ b/nixos/tests/containers-tmpfs.nix @@ -26,7 +26,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { config = { }; }; - virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenv ]; }; testScript = '' From aa22fa9c0b6329e7d557be363c13508a8d46487d Mon Sep 17 00:00:00 2001 From: talyz Date: Fri, 15 Oct 2021 17:22:37 +0200 Subject: [PATCH 27/32] trivial-builders: Add writeStringReferencesToFile Add writeStringReferencesToFile, a builder which extracts a string's references to derivations and paths and writes them to a text file, removing the input string itself from the dependency graph. This is useful when you want to make a derivation depend on the string's references, but not its content (to avoid unnecessary rebuilds, for example). --- pkgs/build-support/trivial-builders.nix | 82 +++++++++++++++++++ pkgs/build-support/trivial-builders/test.nix | 26 ++++-- .../trivial-builders/test/sample.nix | 6 ++ 3 files changed, 109 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index f06d2136b8c6..58cdeb269d58 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -465,6 +465,88 @@ rec { ''; + /* + * Extract a string's references to derivations and paths (its + * context) and write them to a text file, removing the input string + * itself from the dependency graph. This is useful when you want to + * make a derivation depend on the string's references, but not its + * contents (to avoid unnecessary rebuilds, for example). + * + * Note that this only works as intended on Nix >= 2.3. + */ + writeStringReferencesToFile = string: + /* + * The basic operation this performs is to copy the string context + * from `string' to a second string and wrap that string in a + * derivation. However, that alone is not enough, since nothing in the + * string refers to the output paths of the derivations/paths in its + * context, meaning they'll be considered build-time dependencies and + * removed from the wrapper derivation's closure. Putting the + * necessary output paths in the new string is however not very + * straightforward - the attrset returned by `getContext' contains + * only references to derivations' .drv-paths, not their output + * paths. In order to "convert" them, we try to extract the + * corresponding paths from the original string using regex. + */ + let + # Taken from https://github.com/NixOS/nix/blob/130284b8508dad3c70e8160b15f3d62042fc730a/src/libutil/hash.cc#L84 + nixHashChars = "0123456789abcdfghijklmnpqrsvwxyz"; + context = builtins.getContext string; + derivations = lib.filterAttrs (n: v: v ? outputs) context; + # Objects copied from outside of the store, such as paths and + # `builtins.fetch*`ed ones + sources = lib.attrNames (lib.filterAttrs (n: v: v ? path) context); + packages = + lib.mapAttrs' + (name: value: + { + inherit value; + name = lib.head (builtins.match "${builtins.storeDir}/[${nixHashChars}]+-(.*)\.drv" name); + }) + derivations; + # The syntax of output paths differs between outputs named `out` + # and other, explicitly named ones. For explicitly named ones, + # the output name is suffixed as `-name`, but `out` outputs + # aren't suffixed at all, and thus aren't easily distinguished + # from named output paths. Therefore, we find all the named ones + # first so we can use them to remove false matches when looking + # for `out` outputs (see the definition of `outputPaths`). + namedOutputPaths = + lib.flatten + (lib.mapAttrsToList + (name: value: + (map + (output: + lib.filter + lib.isList + (builtins.split "(${builtins.storeDir}/[${nixHashChars}]+-${name}-${output})" string)) + (lib.remove "out" value.outputs))) + packages); + # Only `out` outputs + outputPaths = + lib.flatten + (lib.mapAttrsToList + (name: value: + if lib.elem "out" value.outputs then + lib.filter + (x: lib.isList x && + # If the matched path is in `namedOutputPaths`, + # it's a partial match of an output path where + # the output name isn't `out` + lib.all (o: !lib.hasPrefix (lib.head x) o) namedOutputPaths) + (builtins.split "(${builtins.storeDir}/[${nixHashChars}]+-${name})" string) + else + []) + packages); + allPaths = lib.concatStringsSep "\n" (lib.unique (sources ++ namedOutputPaths ++ outputPaths)); + allPathsWithContext = builtins.appendContext allPaths context; + in + if builtins ? getContext then + writeText "string-references" allPathsWithContext + else + writeDirectReferencesToFile (writeText "string-file" string); + + /* Print an error message if the file with the specified name and * hash doesn't exist in the Nix store. This function should only * be used by non-redistributable software with an unfree license diff --git a/pkgs/build-support/trivial-builders/test.nix b/pkgs/build-support/trivial-builders/test.nix index 204fb54fca3d..420a0fd0114d 100644 --- a/pkgs/build-support/trivial-builders/test.nix +++ b/pkgs/build-support/trivial-builders/test.nix @@ -38,11 +38,27 @@ nixosTest { DIRECT_REFS = invokeSamples ./test/invoke-writeDirectReferencesToFile.nix; }; }; - testScript = '' - machine.succeed(""" - ${./test.sh} 2>/dev/console - """) - ''; + testScript = + let + sample = import ./test/sample.nix { inherit pkgs; }; + samplePaths = lib.unique (lib.attrValues sample); + sampleText = pkgs.writeText "sample-text" (lib.concatStringsSep "\n" samplePaths); + stringReferencesText = + pkgs.writeStringReferencesToFile + ((lib.concatMapStringsSep "fillertext" + (d: "${d}") + (lib.attrValues sample)) + '' + STORE=${builtins.storeDir};\nsystemctl start bar-foo.service + ''); + in '' + machine.succeed(""" + ${./test.sh} 2>/dev/console + """) + machine.succeed(""" + echo >&2 Testing string references... + diff -U3 <(sort ${stringReferencesText}) <(sort ${sampleText}) + """) + ''; meta = { license = lib.licenses.mit; # nixpkgs license maintainers = with lib.maintainers; [ diff --git a/pkgs/build-support/trivial-builders/test/sample.nix b/pkgs/build-support/trivial-builders/test/sample.nix index 27aee6b73dbe..807594d74bb3 100644 --- a/pkgs/build-support/trivial-builders/test/sample.nix +++ b/pkgs/build-support/trivial-builders/test/sample.nix @@ -2,6 +2,7 @@ let inherit (pkgs) figlet + zlib hello writeText ; @@ -9,8 +10,13 @@ in { hello = hello; figlet = figlet; + zlib = zlib; + zlib-dev = zlib.dev; norefs = writeText "hi" "hello"; + norefsDup = writeText "hi" "hello"; helloRef = writeText "hi" "hello ${hello}"; + helloRefDup = writeText "hi" "hello ${hello}"; + path = ./invoke-writeReferencesToFile.nix; helloFigletRef = writeText "hi" "hello ${hello} ${figlet}"; inherit (pkgs) emptyFile From 329a4461a7a1898b3aad13968a232a054422c4fb Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 6 Oct 2021 18:47:56 +0200 Subject: [PATCH 28/32] nixos/testing-python: Copy test script derivations to nodes Make sure the all derivations referenced by the test script are available on the nodes. Accessing these derivations works just fine without this change when using 9p to mount the host's store, but when an image is built (virtualisation.buildRootImage), the dependencies need to be copied to the image. We don't want to copy the script itself, though, since that would trigger unnecessary image rebuilds. --- nixos/lib/testing-python.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index dbba9e4c4452..b1e9bed4a5c1 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -209,11 +209,30 @@ rec { let nodes = qemu_pkg: let + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { nodes = nodes qemu_pkg; } + else testScript; + build-vms = import ./build-vms.nix { inherit system lib pkgs minimal specialArgs; extraConfigurations = extraConfigurations ++ [( { virtualisation.qemu.package = qemu_pkg; + + # Make sure all derivations referenced by the test + # script are available on the nodes. When the store is + # accessed through 9p, this isn't important, since + # everything in the store is available to the guest, + # but when building a root image it is, as all paths + # that should be available to the guest has to be + # copied to the image. + virtualisation.additionalPaths = + lib.optional + (builtins.hasContext testScript') + (pkgs.writeStringReferencesToFile testScript'); + # Ensure we do not use aliases. Ideally this is only set # when the test framework is used by Nixpkgs NixOS tests. nixpkgs.config.allowAliases = false; From 2781f84bce11789a7d8127559a6de0d5cdf92e93 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 6 Oct 2021 10:27:33 +0200 Subject: [PATCH 29/32] discourse.tests: Improve test performance Massively reduce the time it takes running the test by building a proper root disk image and increasing the virtualized core count to 4. This should make it much easier for the tests to pass even on weaker systems. With my laptop (AMD Ryzen 7 PRO 2700U, 32GB RAM) as the reference system, I see the following test run times: - No change: 25 mins, 49 secs - Building a root image: 4 mins, 44 secs - Building a root image and bumping the core count: 3 mins, 6 secs The times include the time it takes to build the image (~40 secs). --- nixos/tests/discourse.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/tests/discourse.nix b/nixos/tests/discourse.nix index 7dd39085a007..cfac5f84a62f 100644 --- a/nixos/tests/discourse.nix +++ b/nixos/tests/discourse.nix @@ -28,6 +28,8 @@ import ./make-test-python.nix ( { nodes, ... }: { virtualisation.memorySize = 2048; + virtualisation.cores = 4; + virtualisation.useNixStoreImage = true; imports = [ common/user-account.nix ]; From ed2497b4050946a148c1efa9688d951dce339b77 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 6 Oct 2021 11:41:16 +0200 Subject: [PATCH 30/32] gitlab.tests: Improve test performance Massively reduce the time it takes running the test by building a proper root disk image and increasing the virtualized core count to 4. This should make it much easier for the tests to pass even on weaker systems. With my laptop (AMD Ryzen 7 PRO 2700U) as the reference system, I see the following test run times: - No change: Times out after 28 mins - Building a root image: 7 mins, 48 secs - Building a root image and bumping the core count: 7 mins, 17 secs The times include the time it takes to build the image (~1 min, 20 secs). --- nixos/tests/gitlab.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab.nix index 3e9feeb0769d..dc3b889c8e8e 100644 --- a/nixos/tests/gitlab.nix +++ b/nixos/tests/gitlab.nix @@ -14,6 +14,8 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; { imports = [ common/user-account.nix ]; virtualisation.memorySize = if pkgs.stdenv.is64bit then 4096 else 2047; + virtualisation.cores = 4; + virtualisation.useNixStoreImage = true; systemd.services.gitlab.serviceConfig.Restart = mkForce "no"; systemd.services.gitlab-workhorse.serviceConfig.Restart = mkForce "no"; systemd.services.gitaly.serviceConfig.Restart = mkForce "no"; From 46f75211446641a5ddb603fbf89b08f30656ebdf Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 20 Oct 2021 17:52:12 +0200 Subject: [PATCH 31/32] make-disk-image: Add an argument to only build a Nix store image Add the `onlyNixStore` argument which enables building images containing the contents of the Nix store at the root, instead of a full system. --- nixos/lib/make-disk-image.nix | 49 +++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix index 9bbfc6c490e3..0a4a71fadc42 100644 --- a/nixos/lib/make-disk-image.nix +++ b/nixos/lib/make-disk-image.nix @@ -51,7 +51,7 @@ fsType ? "ext4" , # Filesystem label - label ? "nixos" + label ? if onlyNixStore then "nix-store" else "nixos" , # The initial NixOS configuration file to be copied to # /etc/nixos/configuration.nix. @@ -60,6 +60,11 @@ , # Shell code executed after the VM has finished. postVM ? "" +, # Copy the contents of the Nix store to the root of the image and + # skip further setup. Incompatible with `contents`, + # `installBootLoader` and `configFile`. + onlyNixStore ? false + , name ? "nixos-disk-image" , # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw. @@ -83,6 +88,7 @@ assert lib.all (attrs: ((attrs.user or null) == null) == ((attrs.group or null) == null)) contents; +assert onlyNixStore -> contents == [] && configFile == null && !installBootLoader; with lib; @@ -345,25 +351,29 @@ let format' = format; in let ''} echo "copying staging root to image..." - cptofs -p ${optionalString (partitionTableType != "none") "-P ${rootPartition}"} -t ${fsType} -i $diskImage $root/* / || + cptofs -p ${optionalString (partitionTableType != "none") "-P ${rootPartition}"} \ + -t ${fsType} \ + -i $diskImage \ + $root${optionalString onlyNixStore builtins.storeDir}/* / || (echo >&2 "ERROR: cptofs failed. diskSize might be too small for closure."; exit 1) ''; -in pkgs.vmTools.runInLinuxVM ( - pkgs.runCommand name - { preVM = prepareImage; + + moveOrConvertImage = '' + ${if format == "raw" then '' + mv $diskImage $out/${filename} + '' else '' + ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${format} ${compress} $diskImage $out/${filename} + ''} + diskImage=$out/${filename} + ''; + + buildImage = pkgs.vmTools.runInLinuxVM ( + pkgs.runCommand name { + preVM = prepareImage; buildInputs = with pkgs; [ util-linux e2fsprogs dosfstools ]; - postVM = '' - ${if format == "raw" then '' - mv $diskImage $out/${filename} - '' else '' - ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${format} ${compress} $diskImage $out/${filename} - ''} - diskImage=$out/${filename} - ${postVM} - ''; + postVM = moveOrConvertImage + postVM; memSize = 1024; - } - '' + } '' export PATH=${binPath}:$PATH rootDisk=${if partitionTableType != "none" then "/dev/vda${rootPartition}" else "/dev/vda"} @@ -425,4 +435,9 @@ in pkgs.vmTools.runInLinuxVM ( tune2fs -T now -c 0 -i 0 $rootDisk ''} '' -) + ); +in + if onlyNixStore then + pkgs.runCommand name {} + (prepareImage + moveOrConvertImage + postVM) + else buildImage From 33b3dae3480526f03065310321e813a5f466b078 Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 28 Oct 2021 12:23:25 +0200 Subject: [PATCH 32/32] rl-2111: Note the addition of virtualisation.useNixStoreImage... ...and rename of virtualisation.pathsInNixDB. --- .../from_md/release-notes/rl-2111.section.xml | 27 ++++++++++++++----- .../manual/release-notes/rl-2111.section.md | 12 ++++++--- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index 77830676e7da..aabe17cb328b 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -1176,6 +1176,13 @@ Superuser created successfully. dropped because it was EOLed by upstream in 2021-10. + + + The virtualisation.pathsInNixDB option was + renamed + virtualisation.additionalPaths. + +
@@ -1199,25 +1206,31 @@ Superuser created successfully. In NixOS virtual machines (QEMU), the virtualisation module has been updated with - new options to configure: + new options: - IPv4 port forwarding - (virtualisation.forwardPorts), + forwardPorts + to configure IPv4 port forwarding, - shared host directories - (virtualisation.sharedDirectories), + sharedDirectories + to set up shared host directories, - screen resolution - (virtualisation.resolution). + resolution + to set the screen resolution, + + + + + useNixStoreImage + to use a disk image for the Nix store instead of 9P. diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 4a47a50b3658..e500d77fa5da 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -362,16 +362,20 @@ In addition to numerous new and upgraded packages, this release has the followin - Nextcloud 20 (`pkgs.nextcloud20`) has been dropped because it was EOLed by upstream in 2021-10. +- The `virtualisation.pathsInNixDB` option was renamed + [`virtualisation.additionalPaths`](options.html#opt-virtualisation.additionalPaths). + ## Other Notable Changes {#sec-release-21.11-notable-changes} - The linux kernel package infrastructure was moved out of `all-packages.nix`, and restructured. Linux related functions and attributes now live under the `pkgs.linuxKernel` attribute set. In particular the versioned `linuxPackages_*` package sets (such as `linuxPackages_5_4`) and kernels from `pkgs` were moved there and now live under `pkgs.linuxKernel.packages.*`. The unversioned ones (such as `linuxPackages_latest`) remain untouched. -- In NixOS virtual machines (QEMU), the `virtualisation` module has been updated with new options to configure: - - IPv4 port forwarding ([`virtualisation.forwardPorts`](options.html#opt-virtualisation.forwardPorts)), - - shared host directories ([`virtualisation.sharedDirectories`](options.html#opt-virtualisation.sharedDirectories)), - - screen resolution ([`virtualisation.resolution`](options.html#opt-virtualisation.resolution)). +- In NixOS virtual machines (QEMU), the `virtualisation` module has been updated with new options: + - [`forwardPorts`](options.html#opt-virtualisation.forwardPorts) to configure IPv4 port forwarding, + - [`sharedDirectories`](options.html#opt-virtualisation.sharedDirectories) to set up shared host directories, + - [`resolution`](options.html#opt-virtualisation.resolution) to set the screen resolution, + - [`useNixStoreImage`](options.html#opt-virtualisation.useNixStoreImage) to use a disk image for the Nix store instead of 9P. In addition, the default [`msize`](options.html#opt-virtualisation.msize) parameter in 9P filesystems (including /nix/store and all shared directories) has been increased to 16K for improved performance.