diff --git a/pkgs/development/compilers/llvm/9/llvm.nix b/pkgs/development/compilers/llvm/9/llvm.nix
index afca35822a47..ce53d5f9b5bd 100644
--- a/pkgs/development/compilers/llvm/9/llvm.nix
+++ b/pkgs/development/compilers/llvm/9/llvm.nix
@@ -55,7 +55,7 @@ in stdenv.mkDerivation (rec {
 
   propagatedBuildInputs = [ ncurses zlib ];
 
-  patches = [
+  patches = stdenv.lib.optionals stdenv.isx86_64 [ # minimize rebuilds, for now at least
     # Force a test to evaluate the saved benchmark for a CPU for which LLVM has
     # an execution model. See NixOS/nixpkgs#119673.
     ../exegesis-force-bdver2.patch