lib.isFunction: fix correctness when taking nested functors

Restore the correct behaviour prior to commit 77334757d7
("lib.isFunction: save an attribute lookup")
while preserving its performance gain.

lib/tests/misc.nix

@@ -986,6 +986,61 @@ runTests {
     expr = isFunction { __functor = null; };
     expected = false;
   };
+  testIsFunctionAttrsWithNestedFunctors = {
+    expr = {
+      a = isFunction { __functor = self: { __functor = self: x: null; }; };
+      b = isFunction {
+        __functor = {
+          __functor =
+            self: outer: x:
+            null;
+        };
+      };
+      c = isFunction {
+        __functor = {
+          __functor = self: outer: { __functor = self: x: null; };
+        };
+      };
+    };
+    expected = {
+      a = true;
+      b = true;
+      c = true;
+    };
+  };
+  testIsFunctionNestedSetFunctionArgs =
+    let
+      faOrig = {
+        x = false;
+      };
+      fa1 = faOrig // {
+        y1 = true;
+      };
+      fa2 = faOrig // {
+        y2 = true;
+      };
+      fa3 = faOrig // {
+        y3 = true;
+      };
+      fa4 = faOrig // {
+        y4 = true;
+      };
+      f0 = { x, ... }: x;
+      f1 = setFunctionArgs f0 fa1;
+      f2 = setFunctionArgs f1 fa2;
+      f3 = setFunctionArgs f2 fa3;
+      f4 = setFunctionArgs f3 fa4;
+    in
+    {
+      expr = map isFunction [
+        f0
+        f1
+        f2
+        f3
+        f4
+      ];
+      expected = replicate 5 true;
+    };
 
   testIsStorePath = {
     expr =

lib/trivial.nix

@@ -1126,9 +1126,21 @@ in
   */
   isFunction =
     let
-      isFunction = builtins.isFunction;
+      isLambda = builtins.isFunction;
     in
-    f: isFunction f || (f ? __functor && isFunction f.__functor && isFunction (f.__functor f));
+    f:
+    # Equivalent to `isLambda f || (f ? __functor && isFunction f.__functor && isFunction (f.__functor f))`
+    # but faster when `f.__functor` and `f.__functor f` are both lambdas.
+    isLambda f
+    || (
+      f ? __functor
+      && (
+        let
+          fFunctor = f.__functor;
+        in
+        (isLambda fFunctor || isFunction fFunctor) && (isLambda (fFunctor f) || isFunction (fFunctor f))
+      )
+    );
 
   /**
     `mirrorFunctionArgs f g` creates a new function `g'` with the same behavior as `g` (`g' x == g x`)