fix rclone sync on b2, add extraArgs, fixed misskey media sync

hosts/hetzner-vm/containers/storage/profiles/rclone-serve.nix

@@ -6,6 +6,7 @@ in {
     "d /caches - storage storage"
     "d /caches/main_webdav_serve - storage storage"
     "d /caches/media_webdav_serve - storage storage"
+    "d /caches/backups_misskey_webdav_serve - storage storage"
   ];
 
   services.rclone-serve = let
@@ -37,6 +38,10 @@ in {
           "--addr=0.0.0.0:${toString ports.rclone_serve_webdav_misskey}"
           "--htpasswd=${secrets.webdav_misskey_htpasswd.path}"
           "--baseurl=/Misskey/"
+          "--cache-dir=/caches/backups_misskey_webdav_serve"
+          "--vfs-cache-max-age=30m"
+          "--vfs-cache-max-size=3g"
+          "--vfs-cache-mode=full"
         ];
         inherit serviceConfig;
       }

hosts/hetzner-vm/containers/storage/profiles/rclone-sync.nix

@@ -6,6 +6,9 @@
         OnStartupSec = "60";
         OnCalendar = "4h";
       };
+      extraArgs = [
+        "--fast-list"
+      ];
     };
   in {
     enable = true;

hosts/hetzner-vm/containers/storage/storage.nix

@@ -48,6 +48,8 @@ in {
           profiles.users
         ]);
 
+      environment.systemPackages = with pkgs; [rclone];
+
       home-manager.users.root = {
         imports = with tree; [home.base home.dev.small];
         home.packages = with pkgs; [vault];

hosts/hetzner-vm/profiles/misskey.nix

@@ -134,19 +134,29 @@ in {
     "L /home/misskey/.config/rclone/rclone.conf - - - - ${secrets.misskey_storage_rclone_config.path}"
   ];
 
-  services.rclone-sync = {
+  systemd.services."misskey-files-sync" = {
-    enable = true;
+    serviceConfig.Type = "oneshot";
-    user = "misskey";
+    script = let
-    sync_jobs = [
+      bsdtar = "${pkgs.libarchive}/bin/bsdtar";
-      {
+      rclone = "${pkgs.rclone}/bin/rclone";
-        source = "/home/misskey/misskey-files";
+    in ''
-        dest = "Storage-Media-Crypt:";
+      pushd /home/misskey
-        id = "misskey_media";
+      pushd /home/misskey/misskey-files
+      ${bsdtar} cvf ../Media.tar .
+      popd
+      ${rclone} copy Media.tar Storage-Media-Crypt:Media.tar
+      rm Media.tar
+      popd
+    '';
+    serviceConfig.User = "misskey";
+  };
+
+  systemd.timers."misskey-files-sync" = {
+    wantedBy = ["timers.target"];
+    partOf = ["misskey-files-sync.service"];
     timerConfig = {
       OnStartupSec = "60";
       OnCalendar = "4h";
     };
-      }
-    ];
   };
 }

modules/nixos/rclone-sync.nix

@@ -17,7 +17,7 @@ with lib; let
           then "${cfg.user}"
           else "root";
 
-        ExecStart = "${pkgs.rclone}/bin/rclone sync ${sync_config.source} ${sync_config.dest} -P";
+        ExecStart = "${pkgs.rclone}/bin/rclone sync ${sync_config.source} ${sync_config.dest} ${lib.concatStringsSep " " sync_config.extraArgs} -P";
       }
       (lib.mkIf sync_config.autoRestart {
         TimeoutSec = 60;
@@ -47,6 +47,11 @@ in {
             dest = mkOption {type = types.str;};
             id = mkOption {type = types.str;};
 
+            extraArgs = mkOption {
+              type = types.listOf types.str;
+              default = [];
+            };
+
             autoRestart = mkOption {
               type = types.bool;
               default = true;
@@ -76,7 +81,7 @@ in {
         [
           (pkgs.writeShellScriptBin "rclone-sync-all" (lib.concatStringsSep "\n" (map (
               job: ''
-                ${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} -P
+                ${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} ${lib.concatStringsSep " " job.extraArgs} -P $@
               ''
             )
             cfg.sync_jobs)))
@@ -85,7 +90,7 @@ in {
           map (
             job:
               pkgs.writeShellScriptBin "rclone-manual-sync-${job.id}" ''
-                exec ${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} -P
+                exec ${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} ${lib.concatStringsSep " " job.extraArgs} -P $@
               ''
           )
           cfg.sync_jobs

overlay/piped/frontend/yarn.nix

@@ -1,4 +1,10 @@
-{ fetchurl, fetchgit, linkFarm, runCommand, gnutar }: rec {
+{
+  fetchurl,
+  fetchgit,
+  linkFarm,
+  runCommand,
+  gnutar,
+}: rec {
   offline_cache = linkFarm "offline" packages;
   packages = [
     {