lib.isFunction, lib.functionArgs: fix correctness when taking nested functors (#574597)

lib/tests/misc.nix

@@ -57,6 +57,7 @@ let
     foldlAttrs
     foldr
     functionArgs
+    genAttrs
     generators
     genList
     getExe
@@ -930,6 +931,54 @@ runTests {
     expected = "1.2.3";
   };
 
+  testFunctionArgsNestedFunctors =
+    let
+      faOrig = {
+        x = false;
+      };
+      faAlt = {
+        x = false;
+        y = true;
+      };
+      f0 = { x, ... }: x;
+    in
+    {
+      expr = {
+        a = functionArgs {
+          __functor = self: {
+            __functor = self: f0;
+            __functionArgs = faAlt;
+          };
+        };
+        b = functionArgs { __functor = self: { __functor = self: f0; }; };
+        c = functionArgs {
+          __functor = {
+            __functor = self: outer: f0;
+          };
+        };
+        d = functionArgs {
+          __functor = {
+            __functor = self: outer: {
+              __functor = self: f0;
+              __functionArgs = faAlt;
+            };
+          };
+        };
+        e = functionArgs {
+          __functor = {
+            __functor = self: outer: { __functor = self: f0; };
+          };
+        };
+      };
+      expected = {
+        a = faAlt;
+        b = faOrig;
+        c = faOrig;
+        d = faAlt;
+        e = faOrig;
+      };
+    };
+
   testIsFunctionStr = {
     expr = isFunction "a";
     expected = false;
@@ -986,6 +1035,61 @@ runTests {
     expr = isFunction { __functor = null; };
     expected = false;
   };
+  testIsFunctionAttrsWithNestedFunctors = {
+    expr = {
+      a = isFunction { __functor = self: { __functor = self: x: null; }; };
+      b = isFunction {
+        __functor = {
+          __functor =
+            self: outer: x:
+            null;
+        };
+      };
+      c = isFunction {
+        __functor = {
+          __functor = self: outer: { __functor = self: x: null; };
+        };
+      };
+    };
+    expected = {
+      a = true;
+      b = true;
+      c = true;
+    };
+  };
+  testIsFunctionNestedSetFunctionArgs =
+    let
+      faOrig = {
+        x = false;
+      };
+      fa1 = faOrig // {
+        y1 = true;
+      };
+      fa2 = faOrig // {
+        y2 = true;
+      };
+      fa3 = faOrig // {
+        y3 = true;
+      };
+      fa4 = faOrig // {
+        y4 = true;
+      };
+      f0 = { x, ... }: x;
+      f1 = setFunctionArgs f0 fa1;
+      f2 = setFunctionArgs f1 fa2;
+      f3 = setFunctionArgs f2 fa3;
+      f4 = setFunctionArgs f3 fa4;
+    in
+    {
+      expr = map isFunction [
+        f0
+        f1
+        f2
+        f3
+        f4
+      ];
+      expected = replicate 5 true;
+    };
 
   testIsStorePath = {
     expr =

lib/trivial.nix

@@ -1104,9 +1104,22 @@ in
   */
   functionArgs =
     let
-      functionArgs = builtins.functionArgs;
+      functionArgsLambda = builtins.functionArgs;
     in
-    f: if f ? __functor then f.__functionArgs or (functionArgs (f.__functor f)) else functionArgs f;
+    f:
+    if f ? __functor then
+      f.__functionArgs or (
+        let
+          fInner = f.__functor f;
+        in
+        # Peel one recursion to avoid calling functionArgs
+        if fInner ? __functor then
+          fInner.__functionArgs or (functionArgs fInner)
+        else
+          functionArgsLambda fInner
+      )
+    else
+      functionArgsLambda f;
 
   /**
     Check whether something is a function or something
@@ -1126,9 +1139,21 @@ in
   */
   isFunction =
     let
-      isFunction = builtins.isFunction;
+      isLambda = builtins.isFunction;
     in
-    f: isFunction f || (f ? __functor && isFunction f.__functor && isFunction (f.__functor f));
+    f:
+    # Equivalent to `isLambda f || (f ? __functor && isFunction f.__functor && isFunction (f.__functor f))`
+    # but faster when `f.__functor` and `f.__functor f` are both lambdas.
+    isLambda f
+    || (
+      f ? __functor
+      && (
+        let
+          fFunctor = f.__functor;
+        in
+        (isLambda fFunctor || isFunction fFunctor) && (isLambda (fFunctor f) || isFunction (fFunctor f))
+      )
+    );
 
   /**
     `mirrorFunctionArgs f g` creates a new function `g'` with the same behavior as `g` (`g' x == g x`)