Merge pull request #208351 from gbtb/opencv4_tests_and_blas_provider

pkgs/development/libraries/opencv/4.x.nix

@@ -15,6 +15,7 @@
 , ocl-icd
 , buildPackages
 , qimgv
+, opencv4
 
 , enableJPEG ? true
 , libjpeg
@@ -31,8 +32,8 @@
 , openjpeg
 , enableEigen ? true
 , eigen
-, enableOpenblas ? true
-, openblas
+, enableBlas ? true
+, blas
 , enableContrib ? true
 
 , enableCuda ? (config.cudaSupport or false) && stdenv.hostPlatform.isx86_64
@@ -74,6 +75,7 @@
 , CoreMedia
 , MediaToolbox
 , bzip2
+, callPackage
 }:
 
 let
@@ -93,6 +95,13 @@ let
     sha256 = "sha256-meya0J3RdOIeMM46e/6IOVwrKn3t/c0rhwP2WQaybkE=";
   };
 
+  testDataSrc = fetchFromGitHub {
+    owner = "opencv";
+    repo = "opencv_extra";
+    rev = version;
+    sha256 = "sha256-6hAdJdaUgtRGQanQKuY/q6fcXWXFZ3K/oLbGxvksry0=";
+  };
+
   # Contrib must be built in order to enable Tesseract support:
   buildContrib = enableContrib || enableTesseract || enableOvis;
 
@@ -171,10 +180,10 @@ let
   ade = rec {
     src = fetchurl {
       url = "https://github.com/opencv/ade/archive/${name}";
-      sha256 = "04n9na2bph706bdxnnqfcbga4cyj8kd9s9ni7qyvnpj5v98jwvlm";
+      sha256 = "sha256-TjLRbFbC7MDY9PxIy560ryviBI58cbQwqgc7A7uOHkg=";
     };
-    name = "v0.1.1f.zip";
-    md5 = "b624b995ec9c439cbc2e9e6ee940d3a2";
+    name = "v0.1.2a.zip";
+    md5 = "fa4b3e25167319cb0fa9432ef8281945";
     dst = ".cache/ade";
   };
 
@@ -208,13 +217,24 @@ let
 
   opencvFlag = name: enabled: "-DWITH_${name}=${printEnabled enabled}";
 
+  runAccuracyTests = true;
+  runPerformanceTests = false;
   printEnabled = enabled: if enabled then "ON" else "OFF";
+  withOpenblas = (enableBlas && blas.provider.pname == "openblas");
+  #multithreaded openblas conflicts with opencv multithreading, which manifest itself in hung tests
+  #https://github.com/xianyi/OpenBLAS/wiki/Faq/4bded95e8dc8aadc70ce65267d1093ca7bdefc4c#multi-threaded
+  openblas_ = blas.provider.override { singleThreaded = true; };
 in
 
 stdenv.mkDerivation {
   pname = "opencv";
   inherit version src;
 
+  outputs = [
+    "out"
+    "package_tests"
+  ];
+
   postUnpack = lib.optionalString buildContrib ''
     cp --no-preserve=mode -r "${contribSrc}/modules" "$NIX_BUILD_TOP/source/opencv_contrib"
   '';
@@ -263,12 +283,12 @@ stdenv.mkDerivation {
     ++ lib.optional enableFfmpeg ffmpeg
     ++ lib.optionals (enableFfmpeg && stdenv.isDarwin)
       [ VideoDecodeAcceleration bzip2 ]
-    ++ lib.optionals enableGStreamer (with gst_all_1; [ gstreamer gst-plugins-base ])
+    ++ lib.optionals enableGStreamer (with gst_all_1; [ gstreamer gst-plugins-base gst-plugins-good ])
     ++ lib.optional enableOvis ogre
     ++ lib.optional enableGPhoto2 libgphoto2
     ++ lib.optional enableDC1394 libdc1394
     ++ lib.optional enableEigen eigen
-    ++ lib.optional enableOpenblas openblas
+    ++ lib.optional enableBlas blas.provider
     # There is seemingly no compile-time flag for Tesseract.  It's
     # simply enabled automatically if contrib is built, and it detects
     # tesseract & leptonica.
@@ -290,7 +310,8 @@ stdenv.mkDerivation {
   NIX_CFLAGS_COMPILE = lib.optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR";
 
   # Configure can't find the library without this.
-  OpenBLAS_HOME = lib.optionalString enableOpenblas openblas;
+  OpenBLAS_HOME = lib.optionalString withOpenblas openblas_.dev;
+  OpenBLAS = lib.optionalString withOpenblas openblas_;
 
   cmakeFlags = [
     "-DOPENCV_GENERATE_PKGCONFIG=ON"
@@ -299,8 +320,9 @@ stdenv.mkDerivation {
     "-DProtobuf_PROTOC_EXECUTABLE=${lib.getExe buildPackages.protobuf}"
     "-DPROTOBUF_UPDATE_FILES=ON"
     "-DOPENCV_ENABLE_NONFREE=${printEnabled enableUnfree}"
-    "-DBUILD_TESTS=OFF"
-    "-DBUILD_PERF_TESTS=OFF"
+    "-DBUILD_TESTS=${printEnabled runAccuracyTests}"
+    "-DBUILD_PERF_TESTS=${printEnabled runPerformanceTests}"
+    "-DCMAKE_SKIP_BUILD_RPATH=ON"
     "-DBUILD_DOCS=${printEnabled enableDocs}"
     # "OpenCV disables pkg-config to avoid using of host libraries. Consider using PKG_CONFIG_LIBDIR to specify target SYSROOT"
     # but we have proper separation of build and host libs :), fixes cross
@@ -334,6 +356,14 @@ stdenv.mkDerivation {
     make doxygen
   '';
 
+  preInstall =
+    lib.optionalString (runAccuracyTests || runPerformanceTests) ''
+    mkdir $package_tests
+    cp -R $src/samples $package_tests/
+    ''
+    + lib.optionalString runAccuracyTests "mv ./bin/*test* $package_tests/ \n"
+    + lib.optionalString runPerformanceTests "mv ./bin/*perf* $package_tests/";
+
   # By default $out/lib/pkgconfig/opencv4.pc looks something like this:
   #
   #   prefix=/nix/store/g0wnfyjjh4rikkvp22cpkh41naa43i4i-opencv-4.0.0
@@ -368,16 +398,23 @@ stdenv.mkDerivation {
 
   passthru = {
     tests = {
-      inherit qimgv;
       inherit (gst_all_1) gst-plugins-bad;
-    } // lib.optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; };
+    }
+    // lib.optionalAttrs (!stdenv.isDarwin) { inherit qimgv; }
+    // lib.optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; }
+    // lib.optionalAttrs (stdenv.buildPlatform != "x86_64-darwin") {
+      opencv4-tests = callPackage ./tests.nix {
+        inherit enableGStreamer enableGtk2 enableGtk3 runAccuracyTests runPerformanceTests testDataSrc;
+        inherit opencv4;
+        };
+      };
   } // lib.optionalAttrs enablePython { pythonPath = [ ]; };
 
   meta = with lib; {
     description = "Open Computer Vision Library with more than 500 algorithms";
     homepage = "https://opencv.org/";
     license = with licenses; if enableUnfree then unfree else bsd3;
-    maintainers = with maintainers; [ mdaiter basvandijk ];
+    maintainers = with maintainers; [ basvandijk ];
     platforms = with platforms; linux ++ darwin;
   };
 }

pkgs/development/libraries/opencv/tests.nix

@@ -0,0 +1,70 @@
+{ opencv4
+, testDataSrc
+, stdenv
+, lib
+, runCommand
+, gst_all_1
+, runAccuracyTests
+, runPerformanceTests
+, enableGStreamer
+, enableGtk2
+, enableGtk3
+, xvfb-run
+}:
+let
+  testNames = [
+    "calib3d"
+    "core"
+    "features2d"
+    "flann"
+    "imgcodecs"
+    "imgproc"
+    "ml"
+    "objdetect"
+    "photo"
+    "stitching"
+    "video"
+    #"videoio" # - a lot of GStreamer warnings and failed tests
+    #"dnn" #- some caffe tests failed, probably because github workflow also downloads additional models
+  ] ++ lib.optionals (!stdenv.isAarch64 && enableGStreamer) [ "gapi" ]
+  ++ lib.optionals (enableGtk2 || enableGtk3) [ "highgui" ];
+  perfTestNames = [
+    "calib3d"
+    "core"
+    "features2d"
+    "imgcodecs"
+    "imgproc"
+    "objdetect"
+    "photo"
+    "stitching"
+    "video"
+  ] ++ lib.optionals (!stdenv.isAarch64 && enableGStreamer) [ "gapi" ];
+  testRunner = if stdenv.isDarwin then "" else "${lib.getExe xvfb-run} -a ";
+  testsPreparation = ''
+    touch $out
+    # several tests want a write access, so we have to copy files
+    tmpPath="$(mktemp -d "/tmp/opencv_extra_XXXXXX")"
+    cp -R ${testDataSrc} $tmpPath/opencv_extra
+    chmod -R +w $tmpPath/opencv_extra
+    export OPENCV_TEST_DATA_PATH="$tmpPath/opencv_extra/testdata"
+    export OPENCV_SAMPLES_DATA_PATH="${opencv4.package_tests}/samples/data"
+
+    #ignored tests because of gtest error - "Test code is not available due to compilation error with GCC 11"
+    export GTEST_FILTER="-AsyncAPICancelation/cancel*"
+  '';
+  accuracyTests = lib.optionalString runAccuracyTests ''
+    ${ builtins.concatStringsSep "\n"
+      (map (test: "${testRunner}${opencv4.package_tests}/opencv_test_${test} --test_threads=$NIX_BUILD_CORES --gtest_filter=$GTEST_FILTER" ) testNames)
+    }
+  '';
+  perfomanceTests = lib.optionalString runPerformanceTests ''
+    ${ builtins.concatStringsSep "\n"
+      (map (test: "${testRunner}${opencv4.package_tests}/opencv_perf_${test} --perf_impl=plain --perf_min_samples=10 --perf_force_samples=10 --perf_verify_sanity --skip_unstable=1 --gtest_filter=$GTEST_FILTER") perfTestNames)
+    }
+  '';
+in
+runCommand "opencv4-tests"
+{
+  nativeBuildInputs = lib.optionals enableGStreamer (with gst_all_1; [ gstreamer gst-plugins-base gst-plugins-good ]);
+}
+  (testsPreparation + accuracyTests + perfomanceTests)