diff --git a/hosts/hetzner-vm/services/quassel.nix b/hosts/hetzner-vm/services/quassel.nix index b77653a..557bb67 100644 --- a/hosts/hetzner-vm/services/quassel.nix +++ b/hosts/hetzner-vm/services/quassel.nix @@ -6,13 +6,13 @@ _: { # needed so quassel can access SSL certs # TODO: set up SSL - users.groups.acme.members = [ "quassel" ]; + # users.groups.acme.members = [ "quassel" ]; # Not gonna serve anything there but shrug. - services.nginx.virtualHosts."quassel.owo.monster" = { - forceSSL = true; - enableACME = true; - }; + #services.nginx.virtualHosts."quassel.owo.monster" = { + # forceSSL = true; + # enableACME = true; + #}; services.postgresql.ensureDatabases = [ "quassel" ]; services.postgresql.ensureUsers = [{ diff --git a/hosts/hetzner-vm/services/restic.nix b/hosts/hetzner-vm/services/restic.nix index 00c7871..88fc7c7 100644 --- a/hosts/hetzner-vm/services/restic.nix +++ b/hosts/hetzner-vm/services/restic.nix @@ -35,7 +35,9 @@ let }; repos = { Chaos-Backups-HetznerVM = { - repository = "b2:Chaos-Backups:HetznerVM"; + # repository is overrided in environmentFile to contain auth + # make sure to keep up to date when changing repository + repository = "rest:https://storage-restic.owo.monster/HetznerVM"; passwordFile = "/secrets/restic-Chaos-Backups-HetznerVM-password"; environmentFile = "/secrets/restic-Chaos-Backups-HetznerVM-env"; }; @@ -51,9 +53,9 @@ let # nya pkgs.writeShellScriptBin "restic-${repoName}" '' env \ - $(cat ${repoInfo.environmentFile}) \ RESTIC_PASSWORD_FILE=${repoInfo.passwordFile} \ RESTIC_REPOSITORY=${repoInfo.repository} \ + $(cat ${repoInfo.environmentFile}) \ ${pkgs.restic}/bin/restic $@ '')) repos; in { diff --git a/hosts/nixos.nix b/hosts/nixos.nix index 648d3a3..c17ce04 100644 --- a/hosts/nixos.nix +++ b/hosts/nixos.nix @@ -47,7 +47,11 @@ in { storage = nixosUnstableSystem { specialArgs = defaultSpecialArgs; system = "x86_64-linux"; - modules = defaultModules ++ [ ./storage/modules/rclone-serve.nix ./storage/storage.nix ]; + modules = defaultModules ++ [ + ./storage/modules/rclone-serve.nix + ./storage/modules/rclone-sync.nix + ./storage/storage.nix + ]; }; # nix --no-sandbox build .#nixosConfigurations.raspberry.config.system.build.sdImage diff --git a/hosts/storage/modules/rclone-serve.nix b/hosts/storage/modules/rclone-serve.nix index bee212b..a1fdf33 100644 --- a/hosts/storage/modules/rclone-serve.nix +++ b/hosts/storage/modules/rclone-serve.nix @@ -7,9 +7,9 @@ let daemonService = serve_config: { enable = true; - requires = [ "network.target" ]; - after = [ "network.target" ] - ++ (if serve_config.after != null then serve_config.after else [ ]); + after = [ "network.target" ]; + wants = [ "network.target" ] + ++ (if serve_config.wants != null then serve_config.wants else [ ]); wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "simple"; @@ -21,7 +21,7 @@ let ExecStart = "${pkgs.rclone}/bin/rclone serve ${serve_config.type} ${serve_config.remote} ${ - lib.concatStrings serve_config.extraArgs + lib.concatStringsSep " " serve_config.extraArgs }"; }; }; @@ -39,7 +39,7 @@ in { remote = mkOption { type = types.str; }; type = mkOption { type = types.str; }; user = mkOption { type = types.str; }; - after = mkOption { type = types.listOf types.str; }; + wants = mkOption { type = types.listOf types.str; }; extraArgs = mkOption { type = types.listOf types.str; }; }; diff --git a/hosts/storage/modules/rclone-sync.nix b/hosts/storage/modules/rclone-sync.nix new file mode 100644 index 0000000..a1db712 --- /dev/null +++ b/hosts/storage/modules/rclone-sync.nix @@ -0,0 +1,73 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.rclone-sync; + + makeNameSafe = name: builtins.replaceStrings [ "/" ":" ] [ "-" "-" ] name; + + daemonService = sync_config: { + serviceConfig = { + Type = "oneshot"; + + User = if cfg.user != null then "${cfg.user}" else "root"; + + ExecStart = + "${pkgs.rclone}/bin/rclone sync ${sync_config.source} ${sync_config.dest} -P"; + }; + }; +in { + options = { + services.rclone-sync = { + enable = mkOption { + type = types.bool; + default = false; + }; + + user = mkOption { + type = types.str; + default = null; + }; + + sync_jobs = mkOption { + type = types.listOf (types.submodule { + options = { + source = mkOption { type = types.str; }; + dest = mkOption { type = types.str; }; + + timerConfig = mkOption { type = types.attrs; }; + + wants = mkOption { + type = types.listOf types.str; + default = [ ]; + }; + }; + }); + default = [ ]; + }; + }; + }; + + config = mkMerge [ + (mkIf (cfg.enable && cfg.sync_jobs != [ ]) { + systemd.services = listToAttrs (map (job: { + name = + "rclone-sync-${makeNameSafe job.source}-${makeNameSafe job.dest}"; + value = daemonService job; + }) cfg.sync_jobs); + systemd.timers = listToAttrs (map (job: { + name = + "rclone-sync-${makeNameSafe job.source}-${makeNameSafe job.dest}"; + value = { + wantedBy = [ "timers.target" ]; + wants = job.wants; + partOf = [ + "rclone-sync-${makeNameSafe job.source}-${ + makeNameSafe job.dest + }.service" + ]; + timerConfig = job.timerConfig; + }; + }) cfg.sync_jobs); + }) + ]; +} diff --git a/hosts/storage/populate-rclone-config.sh b/hosts/storage/populate-rclone-config.sh index 254a7fe..74d3e60 100755 --- a/hosts/storage/populate-rclone-config.sh +++ b/hosts/storage/populate-rclone-config.sh @@ -30,9 +30,15 @@ STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password) sed -i "s/STORAGEBOX_PASSWORD/${STORAGEBOX_PASSWORD}/" ./template B2_CHAOS_BACKUPS_ACCOUNT=$(simple_get /api-keys/backblaze/Chaos-Backups .keyID) -B2_CHAOS_BACKUPS_KEY=$(simple_get /api-keys/backblaze/Chaos-Backups .applicationKey) +B2_CHAOS_BACKUPS_KEY=$(simple_get /api-keys/backblaze/Chaos-Backups .applicationKey | sed "s#/#\\\/#") sed -i "s/B2_CHAOS_BACKUPS_ACCOUNT/${B2_CHAOS_BACKUPS_ACCOUNT}/" ./template sed -i "s/B2_CHAOS_BACKUPS_KEY/${B2_CHAOS_BACKUPS_KEY}/" ./template + +B2_CHAOS_PHOTOS_ACCOUNT=$(simple_get /api-keys/backblaze/Chaos-Photos .keyID) +B2_CHAOS_PHOTOS_KEY=$(simple_get /api-keys/backblaze/Chaos-Photos .applicationKey | sed "s#/#\\\/#") +sed -i "s/B2_CHAOS_PHOTOS_ACCOUNT/${B2_CHAOS_PHOTOS_ACCOUNT}/" ./template +sed -i "s/B2_CHAOS_PHOTOS_KEY/${B2_CHAOS_PHOTOS_KEY}/" ./template + popd cat "${TMP_DIR}/template" > "${OUTPUT_FILE}" diff --git a/hosts/storage/rclone_config.template b/hosts/storage/rclone_config.template index 7dfe73e..167520f 100644 --- a/hosts/storage/rclone_config.template +++ b/hosts/storage/rclone_config.template @@ -16,4 +16,14 @@ hard_delete = true [B2-Chaos-Backups] type = alias -remote = B2-Chaos-Backups-Source:Chaos-Backups \ No newline at end of file +remote = B2-Chaos-Backups-Source:Chaos-Backups + +[B2-Chaos-Photos-Source] +type = b2 +account = B2_CHAOS_PHOTOS_ACCOUNT +key = B2_CHAOS_PHOTOS_KEY +hard_delete = true + +[B2-Chaos-Photos] +type = alias +remote = B2-Chaos-Photos-Source:Chaos-Photos \ No newline at end of file diff --git a/hosts/storage/storage.nix b/hosts/storage/storage.nix index c97b49d..0c884ff 100644 --- a/hosts/storage/storage.nix +++ b/hosts/storage/storage.nix @@ -23,7 +23,6 @@ systemd.services.populate-rclone-config = { wantedBy = [ "multi-user.target" ]; - requires = [ "network.target" ]; after = [ "network.target" ]; path = with pkgs; [ bash rclone vault getent jq ]; script = let @@ -42,24 +41,17 @@ ''; }; - systemd.tmpfiles.rules = [ "d /storage 0755 storage storage -" ]; systemd.services.storage-mount = { wantedBy = [ "multi-user.target" ]; - requires = [ - "network.target" - "populate-rclone-config.service" - "systemd-tmpfiles-setup.service" - ]; - after = [ - "network.target" - "populate-rclone-config.service" - "systemd-tmpfiles-setup.service" - ]; - path = with pkgs; [ bash rclone mount ]; + after = [ "network.target" ]; + wants = [ "populate-rclone-config.service" ]; + + path = with pkgs; [ bash rclone mount umount ]; script = '' set -e - umount /storage || true - rclone --config /home/storage/.config/rclone/rclone.conf mount StorageBox: /storage + umount /storage -fl || true + sleep 2 + rclone --config /home/storage/.config/rclone/rclone.conf mount StorageBox: /storage --allow-non-empty ''; }; @@ -81,22 +73,85 @@ services.rclone-serve = { enable = true; - remotes = [{ - user = "storage"; - remote = "StorageBox:Chaos-Backups/DNS"; - type = "webdav"; - after = [ "populate-rclone-config.service" ]; - extraArgs = [ "--addr=:4242" ]; - }]; + remotes = [ + { + user = "storage"; + remote = "StorageBox:"; + type = "webdav"; + wants = [ "populate-rclone-config.service" ]; + extraArgs = [ "--addr=:4242" "--htpasswd=/secrets/webdav_htpasswd" ]; + } + { + user = "storage"; + remote = "StorageBox:Chaos-Backups/HetznerVM"; + type = "restic"; + wants = [ "populate-rclone-config.service" ]; + extraArgs = [ + "--addr=:4243" + "--htpasswd=/secrets/restic_hetznervm_htpasswd" + "--baseurl=/HetznerVM/" + ]; + } + { + user = "storage"; + remote = "StorageBox:Chaos-Backups/Music"; + type = "restic"; + wants = [ "populate-rclone-config.service" ]; + extraArgs = [ + "--addr=:4244" + "--htpasswd=/secrets/restic_music_htpasswd" + "--baseurl=/Music/" + ]; + } + ]; }; - services.nginx.virtualHosts."storage-web.owo.monster" = { + services.nginx.virtualHosts."storage-webdav.owo.monster" = { forceSSL = true; enableACME = true; - #locations = { "/".proxyPass = "http://localhost:4242"; }; + locations = { "/".proxyPass = "http://localhost:4242"; }; }; - environment.systemPackages = with pkgs; [ rclone cifs-utils ]; + services.nginx.virtualHosts."storage-restic.owo.monster" = { + forceSSL = true; + enableACME = true; + locations = { + "/HetznerVM/".proxyPass = "http://localhost:4243"; + "/Music/".proxyPass = "http://localhost:4244"; + }; + }; + + services.rclone-sync = { + enable = true; + user = "storage"; + sync_jobs = [ + { + source = "StorageBox:Chaos-Backups"; + dest = "B2-Chaos-Backups:"; + wants = [ "populate-rclone-config.service" ]; + timerConfig = { + OnStartupSec = "60"; + OnCalendar = "4h"; + }; + } + { + source = "StorageBox:Chaos-Photos"; + dest = "B2-Chaos-Photos:"; + wants = [ "populate-rclone-config.service" ]; + timerConfig = { + OnStartupSec = "60"; + OnCalendar = "4h"; + }; + } + ]; + }; + + environment.systemPackages = with pkgs; [ + rclone + cifs-utils + apacheHttpd + restic + ]; home-manager.users.root = { imports = with tree; [ home.base home.dev.small ];