openssl: adaptations for combined handshakes and the PQC era

Now that PQC is in OpenSSL 3.5 by default,
handshakes will took longer, as the can combine
a PQC algorithm with a conventional one.

Therefore add the elliptic curve optimization
for x86_64 which Arch Linux is also using by
default (enable-ec_nistp_64_gcc_128).

Furthermore, make the security level configurable
for power users. Setting this to 5 will only allow for
connections with security strength 256 bit. Please
beware, that this may lead to no common cipher and key
lengths (no connection at all). Set to OpenSSL's default
when not set (2 for OpenSSL 3.5).

Signed-off-by: Markus Theil <theil.markus@gmail.com>

pkgs/development/libraries/openssl/default.nix

@@ -15,6 +15,11 @@
   enableSSL3 ? false,
   enableMD2 ? false,
   enableKTLS ? stdenv.hostPlatform.isLinux,
+  # change this to a value between 0 and 5 (as of OpenSSL 3.5)
+  # if null, default is used, changes the permitted algorithms
+  # and key lengths in the default config
+  # see: https://docs.openssl.org/3.5/man3/SSL_CTX_set_security_level/
+  securityLevel ? null,
   static ? stdenv.hostPlatform.isStatic,
   # path to openssl.cnf file. will be placed in $etc/etc/ssl/openssl.cnf to replace the default
   conf ? null,
@@ -27,6 +32,9 @@
 # cgit) that are needed here should be included directly in Nixpkgs as
 # files.
 
+# check from time to time, if this range is still correct
+assert (securityLevel == null) || (securityLevel >= 0 && securityLevel <= 5);
+
 let
   common =
     {
@@ -181,6 +189,15 @@ let
           "-DHAVE_CRYPTODEV"
           "-DUSE_CRYPTODEV_DIGESTS"
         ]
+        # enable optimized EC curve primitives on x86_64,
+        # can provide a 2x up to 4x speedup at best
+        # with combined PQC and conventional crypto handshakes
+        # starting with 3.5 its nice to speed things up for free
+        ++ lib.optional stdenv.hostPlatform.isx86_64 "enable-ec_nistp_64_gcc_128"
+        # useful to set e.g. 256 bit security level with setting this to 5
+        ++ lib.optional (
+          securityLevel != null
+        ) "-DOPENSSL_TLS_SECURITY_LEVEL=${builtins.toString securityLevel}"
         ++ lib.optional enableMD2 "enable-md2"
         ++ lib.optional enableSSL2 "enable-ssl2"
         ++ lib.optional enableSSL3 "enable-ssl3"