rclone: add serve options

modules/programs/rclone.nix

@@ -4,14 +4,143 @@
   pkgs,
   ...
 }:
-
 let
-
   cfg = config.programs.rclone;
   iniFormat = pkgs.formats.ini { };
   replaceIllegalChars = builtins.replaceStrings [ "/" " " "$" ] [ "." "_" "" ];
   isUsingSecretProvisioner = name: config ? "${name}" && config."${name}".secrets != { };
 
+  # serve protocols that can use `Type=notify` services, this is determined from rclone source code
+  serveProtocolNotifies = [
+    "dlna"
+    "http"
+    "restic"
+    "webdav"
+  ];
+
+  # options shared between mounts/serve
+  mountServeOptions = {
+    logLevel = lib.mkOption {
+      type = lib.types.enum [
+        null
+        "ERROR"
+        "NOTICE"
+        "INFO"
+        "DEBUG"
+      ];
+      default = null;
+      example = "INFO";
+      description = ''
+        Set the log level. See <https://rclone.org/docs/#logging> for more.
+      '';
+    };
+    options = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (
+          nullOr (oneOf [
+            bool
+            int
+            float
+            str
+          ])
+        );
+      default = { };
+      apply = lib.mergeAttrs {
+        vfs-cache-mode = "full";
+        cache-dir = "%C/rclone";
+      };
+      description = ''
+        An attribute set of option values passed to the command.
+        To set a boolean option, assign it `true` or `false`. See
+        <https://nixos.org/manual/nixpkgs/stable/#function-library-lib.cli.toCommandLineShellGNU>
+        for more details on the format.
+
+        Some caching options are set by default, namely `vfs-cache-mode = "full"`
+        and `cache-dir`. These can be overridden if desired.
+      '';
+    };
+  };
+
+  # creates a sidecar user service. returns an attrset of systemd services
+  mkRcloneSidecars =
+    # type of the sidecar, either "mounts" or "serve" corresponding to options
+    sidecarType:
+    lib.listToAttrs (
+      lib.concatMap (
+        _remote: # remote name + remote config
+        let
+          remoteName = _remote.name;
+          remote = _remote.value;
+        in
+        lib.concatMap (
+          _sidecar: # sidecar path + sidecar config
+          let
+            # a sidecar is either a mount or a protocol-serve
+            sidecarPath = _sidecar.name;
+            sidecar = _sidecar.value;
+
+            # there are currently only 2 types mount/serve - this may need changed in the future
+            isMount = sidecarType == "mounts";
+            # name of the rclone command to use - also used in service name
+            cmdName = if isMount then "mount" else "serve";
+          in
+          lib.optional sidecar.enable (
+            lib.nameValuePair "rclone-${cmdName}:${replaceIllegalChars sidecarPath}@${remoteName}" {
+              Unit = {
+                Description = "Rclone ${
+                  if isMount then "FUSE daemon" else "protocol serving"
+                } for ${remoteName}:${sidecarPath}";
+                Requires = [ "rclone-config.service" ];
+                After = [ "rclone-config.service" ];
+              };
+
+              Service = {
+                Type =
+                  # all services can be Type=notify except for serve protocols that don't notify
+                  if sidecarType == "serve" && !(builtins.elem sidecar.protocol serveProtocolNotifies) then
+                    "simple"
+                  else
+                    "notify";
+                Environment =
+                  # fusermount/fusermount3
+                  (lib.optional (sidecarType == "mounts") "PATH=/run/wrappers/bin")
+                  ++ lib.optional (sidecar.logLevel != null) "RCLONE_LOG_LEVEL=${sidecar.logLevel}";
+                # rclone exits with code 143 when stopped properly
+                SuccessExitStatus = "143";
+
+                ExecStartPre = lib.mkIf isMount "${pkgs.coreutils}/bin/mkdir -p ${lib.escapeShellArg sidecar.mountPoint}";
+                ExecStart = lib.concatStringsSep " " (
+                  [ "${lib.getExe cfg.package} ${cmdName}" ] # rclone [command]
+                  ++ (
+                    if isMount then
+                      # https://rclone.org/commands/rclone_mount/
+                      [
+                        (lib.cli.toCommandLineShellGNU { } sidecar.options) # [opts]
+                        (lib.escapeShellArg "${remoteName}:${sidecarPath}") # <remote>
+                        (lib.escapeShellArg sidecar.mountPoint) # <mountpoint>
+                      ]
+                    else
+                      # https://rclone.org/commands/rclone_serve/
+                      [
+                        (lib.escapeShellArg sidecar.protocol) # <protocol>
+                        (lib.cli.toCommandLineShellGNU { } sidecar.options) # [opts]
+                        (lib.escapeShellArg "${remoteName}:${sidecarPath}") # <remote>
+                      ]
+                  )
+                );
+                Restart = "on-failure";
+              };
+
+              Install.WantedBy = lib.optional (
+                if isMount then sidecar.autoMount else sidecar.autoStart
+              ) "default.target";
+            }
+          )
+        ) (lib.attrsToList (remote.${sidecarType} or { }))
+      ) (lib.attrsToList cfg.remotes)
+    );
+
 in
 {
   meta.maintainers = with lib.maintainers; [ jess ];
@@ -113,27 +242,10 @@ in
                       options = {
                         enable = lib.mkEnableOption "this mount";
 
-                        autoMount = lib.mkEnableOption "automatic mounting" // {
+                        autoMount = lib.mkEnableOption "automatically mounting the remote on login" // {
                           default = true;
                         };
 
-                        logLevel = lib.mkOption {
-                          type = lib.types.nullOr (
-                            lib.types.enum [
-                              "ERROR"
-                              "NOTICE"
-                              "INFO"
-                              "DEBUG"
-                            ]
-                          );
-                          default = null;
-                          example = "INFO";
-                          description = ''
-                            Set the log-level.
-                            See: https://rclone.org/docs/#logging
-                          '';
-                        };
-
                         mountPoint = lib.mkOption {
                           type = lib.types.str;
                           default = null;
@@ -142,34 +254,8 @@ in
                           '';
                           example = "/home/alice/my-remote";
                         };
-
-                        options = lib.mkOption {
-                          type =
-                            with lib.types;
-                            attrsOf (
-                              nullOr (oneOf [
-                                bool
-                                int
-                                float
-                                str
-                              ])
-                            );
-                          default = { };
-                          apply = lib.mergeAttrs {
-                            vfs-cache-mode = "full";
-                            cache-dir = "%C/rclone";
-                          };
-                          description = ''
-                            An attribute set of option values passed to `rclone mount`. To set
-                            a boolean option, assign it `true` or `false`. See
-                            <https://nixos.org/manual/nixpkgs/stable/#function-library-lib.cli.toCommandLineShellGNU>
-                            for more details on the format.
-
-                            Some caching options are set by default, namely `vfs-cache-mode = "full"`
-                            and `cache-dir`. These can be overridden if desired.
-                          '';
-                        };
-                      };
+                      }
+                      // mountServeOptions;
                     }
                   );
                 default = { };
@@ -199,6 +285,67 @@ in
                 '';
 
               };
+
+              serve = lib.mkOption {
+                type =
+                  with lib.types;
+                  attrsOf (
+                    lib.types.submodule {
+                      options = {
+                        enable = lib.mkEnableOption "serving this path";
+
+                        protocol = lib.mkOption {
+                          type = lib.types.enum [
+                            "dlna"
+                            "docker"
+                            "ftp"
+                            "http"
+                            "nfs"
+                            "restic"
+                            "s3"
+                            "sftp"
+                            "webdav"
+                          ];
+                          description = ''
+                            The protocol to use when serving this path.
+                            See <https://rclone.org/commands/rclone_serve> for more.
+                          '';
+                          example = "http";
+                        };
+
+                        autoStart = lib.mkEnableOption "automatically serving the remote on login" // {
+                          default = true;
+                        };
+                      }
+                      // mountServeOptions;
+                    }
+                  );
+                default = { };
+                description = ''
+                  An attribute set mapping remote file paths to their corresponding serve configurations.
+
+                  For each entry, to perform the equivalent of
+                  `rclone serve protocol remote:path/to/files` — as described in the
+                  rclone documentation <https://rclone.org/commands/rclone_serve/> — we create
+                  a key-value pair like this:
+                  `"path/to/files/on/remote" = { ... }`.
+                '';
+                example = lib.literalExpression ''
+                  {
+                    "path/to/files" = {
+                      enable = true;
+                      protocol = "http";
+                      options = {
+                        addr = "127.0.0.1:3000";
+                        dir-cache-time = "5000h";
+                        poll-interval = "10s";
+                        umask = "002";
+                        user-agent = "Laptop";
+                      };
+                    };
+                  }
+                '';
+              };
             };
           }
         );
@@ -345,66 +492,13 @@ in
             Install.WantedBy = [ "default.target" ];
           };
         };
-
-      mountServices = lib.listToAttrs (
-        lib.concatMap
-          (
-            { name, value }:
-            let
-              remote-name = name;
-              remote = value;
-            in
-            lib.concatMap (
-              { name, value }:
-              let
-                mount-path = name;
-                mount = value;
-              in
-              lib.optional mount.enable (
-                lib.nameValuePair "rclone-mount:${replaceIllegalChars mount-path}@${remote-name}" {
-                  Unit = {
-                    Description = "Rclone FUSE daemon for ${remote-name}:${mount-path}";
-                    Requires = [ "rclone-config.service" ];
-                    After = [ "rclone-config.service" ];
-                  };
-
-                  Service = {
-                    Type = "notify";
-                    Environment = [
-                      # fusermount/fusermount3
-                      "PATH=/run/wrappers/bin"
-                    ]
-                    ++ lib.optional (mount.logLevel != null) "RCLONE_LOG_LEVEL=${mount.logLevel}";
-
-                    ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${lib.escapeShellArg mount.mountPoint}";
-                    ExecStart = lib.concatStringsSep " " [
-                      (lib.getExe cfg.package)
-                      "mount"
-                      (lib.cli.toCommandLineShellGNU { } mount.options)
-                      (lib.escapeShellArg "${remote-name}:${mount-path}")
-                      (lib.escapeShellArg mount.mountPoint)
-                    ];
-                    Restart = "on-failure";
-                  };
-
-                  Install.WantedBy = lib.optional mount.autoMount "default.target";
-                }
-              )
-            ) (lib.attrsToList remote.mounts)
-          )
-          (
-            lib.pipe cfg.remotes [
-              lib.attrsToList
-              (lib.filter (rem: rem.value ? mounts))
-            ]
-          )
-      );
     in
     lib.mkIf cfg.enable {
       home.packages = [ cfg.package ];
       systemd.user.services = lib.mkMerge [
         rcloneConfigService
-        mountServices
+        (mkRcloneSidecars "mounts")
+        (mkRcloneSidecars "serve")
       ];
     };
 }

tests/integration/standalone/rclone/default.nix

@@ -24,6 +24,7 @@ in
     ./secrets-arbitrary-characters.nix
     ./no-type.nix
     ./mount.nix
+    ./serve.nix
     ./shell.nix
     ./atomic.nix
     ./write-after.nix

tests/integration/standalone/rclone/serve.nix

@@ -0,0 +1,81 @@
+{ pkgs, lib, ... }:
+let
+  sshKeys = import "${pkgs.path}/nixos/tests/ssh-keys.nix" pkgs;
+
+  # https://rclone.org/sftp/#ssh-authentication
+  keyPem = lib.pipe sshKeys.snakeOilEd25519PrivateKey.text [
+    lib.trim
+    (lib.replaceStrings [ "\n" ] [ "\\\\n" ])
+  ];
+
+  module = pkgs.writeText "serve-module" ''
+    { pkgs, lib, ... }: {
+      programs.rclone.remotes = {
+        alices-sftp-remote = {
+          config = {
+            type = "sftp";
+            host = "remote";
+            user = "alice";
+            key_pem = "${keyPem}";
+            known_hosts = "${sshKeys.snakeOilEd25519PublicKey}";
+          };
+          serve = {
+            "/home/alice/files" = {
+              enable = true;
+              protocol = "http";
+              options.addr = "localhost:8080";
+            };
+          };
+        };
+      };
+    }
+  '';
+in
+{
+  nodes.remote = {
+    services.openssh.enable = true;
+
+    users.users.alice.openssh.authorizedKeys.keys = [
+      sshKeys.snakeOilEd25519PublicKey
+    ];
+  };
+
+  script = ''
+    remote.wait_for_unit("network.target")
+    remote.wait_for_unit("multi-user.target")
+
+    succeed_as_alice(
+      "mkdir -p /home/alice/.ssh",
+      "install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix"
+    )
+
+    actual = succeed_as_alice("home-manager switch")
+    expected = "rclone-config.service"
+    assert "Starting units: " in actual and expected in actual, \
+      f"expected home-manager switch to contain {expected}, but got {actual}"
+
+    with subtest("Serve a remote over HTTP (sftp)"):
+      # create files on remote
+      succeed_as_alice(
+        "mkdir /home/alice/files",
+        "touch /home/alice/files/other_file",
+        "echo serving > /home/alice/files/test.txt",
+        box=remote
+      )
+
+      # fetch file from server
+      output = succeed_as_alice(
+        "curl -s http://localhost:8080/test.txt"
+      )
+      expected = "serving"
+      assert expected in output, \
+        f"HTTP server response does not contain expected content. Got: {output}"
+
+      # verify file listing
+      output = succeed_as_alice(
+        "curl -s http://localhost:8080/"
+      )
+      assert "other_file" in output, \
+        f"HTTP directory listing does not contain other_file. Got: {output}"
+  '';
+}

tests/modules/programs/rclone/basic-configuration.nix

@@ -0,0 +1,11 @@
+_: {
+  programs.rclone = {
+    enable = true;
+    remotes.myremote.config.type = "local";
+  };
+
+  # make sure config service exists
+  nmt.script = ''
+    assertFileExists home-files/.config/systemd/user/rclone-config.service
+  '';
+}

tests/modules/programs/rclone/default.nix

@@ -0,0 +1,6 @@
+{ lib, pkgs, ... }:
+lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+  rclone-basic-configuration = ./basic-configuration.nix;
+  rclone-mount-service-generation = ./mount-service-generation.nix;
+  rclone-serve-service-generation = ./serve-service-generation.nix;
+}

tests/modules/programs/rclone/mount-service-generation.nix

@@ -0,0 +1,44 @@
+_: {
+  programs.rclone = {
+    enable = true;
+    remotes.sftp-remote = {
+      config = {
+        type = "sftp";
+        host = "backup-server.example.com";
+        user = "alice";
+        key_file = "/home/alice/.ssh/id_ed25519";
+      };
+      mounts = {
+        "documents/work" = {
+          enable = true;
+          mountPoint = "/home/alice/mounts/work-docs";
+          logLevel = "INFO";
+          options = {
+            dir-cache-time = "5000h";
+            poll-interval = "10s";
+            umask = "002";
+          };
+        };
+        "disabled-mount" = {
+          enable = false;
+          mountPoint = "/home/alice/mounts/disabled";
+        };
+      };
+    };
+  };
+
+  nmt.script = ''
+    # test work documents mount
+    service="home-files/.config/systemd/user/rclone-mount:documents.work@sftp-remote.service"
+    assertFileExists "$service"
+    assertFileContains "$service" "rclone mount '--cache-dir=%C/rclone' '--dir-cache-time=5000h' '--poll-interval=10s' '--umask=002' '--vfs-cache-mode=full' sftp-remote:documents/work /home/alice/mounts/work-docs"
+    assertFileContains "$service" "mkdir -p /home/alice/mounts/work-docs"
+    assertFileContains "$service" "RCLONE_LOG_LEVEL=INFO"
+    assertFileContains "$service" "PATH=/run/wrappers/bin"
+    assertFileContains "$service" "Rclone FUSE daemon for sftp-remote:documents/work"
+    assertFileContains "$service" "Type=notify"
+
+    # make sure disabled mount isn't created
+    assertPathNotExists "home-files/.config/systemd/user/rclone-mount:disabled-mount@sftp-remote.service"
+  '';
+}

tests/modules/programs/rclone/serve-service-generation.nix

@@ -0,0 +1,53 @@
+_: {
+  programs.rclone = {
+    enable = true;
+    remotes = {
+      sftp-remote = {
+        config = {
+          type = "sftp";
+          host = "backup-server.example.com";
+          user = "alice";
+          key_file = "/home/alice/.ssh/id_ed25519";
+        };
+        serve = {
+          "documents/work" = {
+            enable = true;
+            protocol = "http";
+            logLevel = "ERROR";
+            options = {
+              addr = "127.0.0.1:8080";
+              dir-cache-time = "5000h";
+            };
+          };
+          "/games" = {
+            enable = true;
+            protocol = "ftp";
+          };
+          "disabled-serve" = {
+            enable = false;
+            protocol = "ftp";
+          };
+        };
+      };
+    };
+  };
+
+  nmt.script = ''
+    # test work documents serve
+    service="home-files/.config/systemd/user/rclone-serve:documents.work@sftp-remote.service"
+    assertFileExists "$service"
+    assertFileContains "$service" "rclone serve http '--addr=127.0.0.1:8080' '--cache-dir=%C/rclone' '--dir-cache-time=5000h' '--vfs-cache-mode=full' sftp-remote:documents/work"
+    assertFileContains "$service" "RCLONE_LOG_LEVEL=ERROR"
+    assertFileContains "$service" "Rclone protocol serving for sftp-remote:documents/work"
+    assertFileContains "$service" "Type=notify"
+
+    # ftp protocol should not have Type=notify
+    service2="home-files/.config/systemd/user/rclone-serve:.games@sftp-remote.service"
+    assertFileExists "$service2"
+    assertFileContains "$service2" "rclone serve ftp '--cache-dir=%C/rclone' '--vfs-cache-mode=full' sftp-remote:/games"
+    assertFileContains "$service2" "Type=simple"
+
+    # make sure disabled serve is not created
+    assertPathNotExists "home-files/.config/systemd/user/rclone-serve:disabled-serve@sftp-remote.service"
+  '';
+}