diff --git a/pkgs/tools/networking/openssh/common.nix b/pkgs/tools/networking/openssh/common.nix
index deeab329d6cd..d93f51d24cfd 100644
--- a/pkgs/tools/networking/openssh/common.nix
+++ b/pkgs/tools/networking/openssh/common.nix
@@ -16,7 +16,6 @@
   # package without splicing See: https://github.com/NixOS/nixpkgs/pull/107606
   pkgs,
   fetchurl,
-  fetchpatch,
   autoreconfHook,
   withAudit ? false,
   audit,
@@ -72,37 +71,6 @@ stdenv.mkDerivation (finalAttrs: {
     # See discussion in https://github.com/NixOS/nixpkgs/pull/16966
     ./dont_create_privsep_path.patch
   ]
-  ++ lib.optionals (lib.versionOlder finalAttrs.version "10.3") [
-    # See discussion in https://github.com/NixOS/nixpkgs/issues/466049 and
-    # https://gitlab.archlinux.org/archlinux/packaging/packages/openssh/-/issues/23
-    (fetchpatch {
-      name = "pkcs11-fetchkey-error-to-debug.patch";
-      url = "https://github.com/openssh/openssh-portable/commit/607f337637f2077b34a9f6f96fc24237255fe175.patch";
-      hunks = [ "2-" ];
-      hash = "sha256-rdvKL6/rwrdhGKlcmdy6fxVgJgaaRsmngX0KkShXAhQ=";
-    })
-    (fetchpatch {
-      name = "pkcs11-fix-pinentry.patch";
-      url = "https://github.com/openssh/openssh-portable/commit/434ba7684054c0637ce8f2486aaacafe65d9b8aa.patch";
-      # only applies to Makefile.in (which doesn't have a date header) so no hunks= needed
-      hash = "sha256-3JQ3IJurngXclORrfC2Bx7xvmGA6w2nIh+eZ0zd0bLY=";
-    })
-
-    # See discussion in https://github.com/NixOS/nixpkgs/issues/453782 and
-    # https://github.com/openssh/openssh-portable/pull/602
-    (fetchpatch {
-      name = "pkcs11-tests-allow-module-path.patch";
-      url = "https://github.com/openssh/openssh-portable/commit/5e7c3f33b2693b668ecfbac84b85f2c0c84410c2.patch";
-      hunks = [ "2-" ];
-      hash = "sha256-mGpRGXurg8K9Wp8qoojG5MQ+3sZW2XKy2z0RDXLHaEc=";
-    })
-    (fetchpatch {
-      name = "ssh-agent-tests-increase-timeout.patch";
-      url = "https://github.com/openssh/openssh-portable/commit/1fdc3c61194819c16063dc430eeb84b81bf42dcf.patch";
-      hunks = [ "2-" ];
-      hash = "sha256-b9YCOav32kY5VEvIG3W1fyD87HaQxof6Zwq9Oo+/Lac=";
-    })
-  ]
   ++ extraPatches;
 
   postPatch =
diff --git a/pkgs/tools/networking/openssh/default.nix b/pkgs/tools/networking/openssh/default.nix
index f8f82a93a828..f128f5e7b700 100644
--- a/pkgs/tools/networking/openssh/default.nix
+++ b/pkgs/tools/networking/openssh/default.nix
@@ -14,11 +14,11 @@ in
 {
   openssh = common rec {
     pname = "openssh";
-    version = "10.3p1";
+    version = "10.4p1";
 
     src = fetchurl {
       url = urlFor version;
-      hash = "sha256-VmgqNruS3PS08Bb9jsjnQFm3mo3iXBXWcNcx59GORfQ=";
+      hash = "sha256-72Am3SrqjVYFljjV0yYpAsiSzrqfiDlYNeDQbT+2Mjg=";
     };
 
     extraPatches = [
@@ -94,21 +94,27 @@ in
 
   openssh_gssapi = common rec {
     pname = "openssh-with-gssapi";
-    version = "10.3p1";
+    version = "10.4p1";
     extraDesc = " with GSSAPI support";
 
     src = fetchurl {
       url = urlFor version;
-      hash = "sha256-VmgqNruS3PS08Bb9jsjnQFm3mo3iXBXWcNcx59GORfQ=";
+      hash = "sha256-72Am3SrqjVYFljjV0yYpAsiSzrqfiDlYNeDQbT+2Mjg=";
     };
 
     extraPatches = [
       ./ssh-keysign-8.5.patch
 
       (fetchpatch {
-        name = "openssh-gssapi.patch";
-        url = "https://salsa.debian.org/ssh-team/openssh/raw/debian/1%2510.3p1-1/debian/patches/gssapi.patch";
-        hash = "sha256-gs5Vw4f/TDxmme1DbrtgwvWcPGGmYIWE/A4JWa551zA=";
+        name = "servconf-fix-gssapi.patch";
+        url = "https://salsa.debian.org/ssh-team/openssh/raw/debian/1%2510.4p1-1/debian/patches/servconf-fix-gssapi.patch";
+        hash = "sha256-ypyaoEhwxo7SYVpjMkCQnrcFgY2ouWJQlrbJy50Lidk=";
+      })
+
+      (fetchpatch {
+        name = "gssapi.patch";
+        url = "https://salsa.debian.org/ssh-team/openssh/raw/debian/1%2510.4p1-1/debian/patches/gssapi.patch";
+        hash = "sha256-K12AE4C0zMdRdMsRMQCMRIFvN+NhNvCgyt0NDZp7n24=";
       })
     ];