move piped to its own container

hosts/hetzner-vm/containers/piped/data/ports.nix

@@ -0,0 +1,4 @@
+{}: {
+  piped-backend = 3012;
+  piped-proxy = 3013;
+}

hosts/hetzner-vm/containers/piped/piped.nix

@@ -0,0 +1,134 @@
+{
+  tree,
+  lib,
+  inputs,
+  config,
+  pkgs,
+  ...
+}: let
+  container-addresses = import ../../data/container-addresses.nix {};
+  hostIP = container-addresses.host;
+  containerIP = container-addresses.containers.piped;
+
+  ports = import ./data/ports.nix {};
+
+  # Using secrets from Host
+  secrets = config.services.secrets.secrets;
+  secrets_list = [
+    "piped_restic_env"
+    "piped_restic_password"
+  ];
+in {
+  imports = with tree; [
+    # needed so can get nginx defaults for proxy
+    hosts.hetzner-vm.modules.piped
+  ];
+
+  containers.piped = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = hostIP;
+    localAddress = containerIP;
+    bindMounts = lib.mkMerge (lib.forEach secrets_list (secret_name: let
+      path = "${secrets.${secret_name}.path}";
+    in {
+      "${path}" = {
+        hostPath = "${path}";
+      };
+    }));
+
+    config = {
+      config,
+      pkgs,
+      ...
+    }: {
+      _module.args = {
+        inherit inputs;
+        inherit tree;
+        host_secrets = secrets;
+      };
+
+      imports = with tree;
+        [
+          profiles.base
+          inputs.home-manager-unstable.nixosModules.home-manager
+          hosts.hetzner-vm.modules.piped
+
+          profiles.sshd
+
+          modules.nixos.secrets
+
+          users.root
+        ]
+        ++ (with hosts.hetzner-vm.containers.piped; [
+          profiles.piped
+          profiles.restic
+        ]);
+
+      # For Shared Secrets
+      systemd.tmpfiles.rules = [
+        "d ${config.services.secrets.secretsDir} - root root"
+      ];
+
+      networking.firewall = {
+        enable = true;
+        allowedTCPPorts = [22] ++ (lib.attrValues ports);
+      };
+
+      home-manager.users.root = {
+        imports = with tree; [home.base home.dev.small];
+
+        home.stateVersion = "23.05";
+      };
+
+      # Manually configure nameserver. Using resolved inside the container seems to fail
+      # currently
+      environment.etc."resolv.conf".text = "nameserver 8.8.8.8";
+      system.stateVersion = "23.05";
+    };
+  };
+
+  services.nginx.virtualHosts."piped.owo.monster" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/".root = let
+      frontend-package = pkgs.piped-frontend.override {
+        backendDomain = "backend.piped.owo.monster";
+      };
+    in "${frontend-package}/share/piped-frontend";
+    extraConfig = ''
+      try_files $uri $uri/ /index.html;
+    '';
+  };
+
+  services.nginx.virtualHosts."backend.piped.owo.monster" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/" = {
+      proxyPass = "http://${containerIP}:${toString ports.piped-backend}";
+    };
+  };
+
+  services.nginx.virtualHosts."proxy.piped.owo.monster" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/" = {
+      proxyPass = "http://${containerIP}:${toString ports.piped-proxy}";
+      extraConfig =
+        config.services.piped.proxyNginxExtraConfig
+        + ''
+          add_header Cache-Control "public, max-age=604800";
+        '';
+    };
+    locations."~ (/videoplayback|/api/v4/|/api/manifest/)" = {
+      proxyPass = "http://${containerIP}:${toString ports.piped-proxy}";
+      extraConfig =
+        config.services.piped.proxyNginxExtraConfig
+        + ''
+          add_header Cache-Control private always;
+        '';
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [4242];
+}

hosts/hetzner-vm/containers/piped/profiles/piped.nix

@@ -1,11 +1,17 @@
 {...}: let
-  ports = import ../ports.nix {};
+  ports = import ../data/ports.nix {};
 in {
   services.piped = {
     enable = true;
     frontendDomain = "piped.owo.monster";
     backendDomain = "backend.piped.owo.monster";
     proxyDomain = "proxy.piped.owo.monster";
+
+    disableNginx = true;
+    disableFrontend = true;
+
+    # Do not set proxyNginxExtraConfig here as needs be set in outside of container
+
     internalBackendPort = ports.piped-backend;
     internalProxyPort = ports.piped-proxy;
   };

hosts/hetzner-vm/containers/piped/profiles/restic.nix

@@ -0,0 +1,56 @@
+{
+  pkgs,
+  config,
+  lib,
+  host_secrets,
+  ...
+}: let
+  secrets = host_secrets;
+
+  backupPrepareCommand = "${
+    (pkgs.writeShellScriptBin "backupPrepareCommand" ''
+      systemctl start postgresqlBackup-piped --wait
+    '')
+  }/bin/backupPrepareCommand";
+in {
+  environment.systemPackages = with pkgs; [
+    restic
+    (pkgs.writeShellScriptBin "restic-piped" ''
+      env \
+        RESTIC_PASSWORD_FILE=${secrets.piped_restic_password.path} \
+        $(cat ${secrets.piped_restic_env.path}) \
+      ${pkgs.restic}/bin/restic $@
+    '')
+  ];
+
+  services.restic.backups.piped = {
+    user = "root";
+    paths = [
+      "/var/backup/postgresql"
+    ];
+
+    # repository is overrided in environmentFile to contain auth
+    # make sure to keep up to date when changing repository
+    repository = "rest:https://storage-restic.owo.monster/Piped";
+    passwordFile = "${secrets.piped_restic_password.path}";
+    environmentFile = "${secrets.piped_restic_env.path}";
+
+    pruneOpts = [
+      "--keep-last 5"
+    ];
+
+    timerConfig = {
+      OnBootSec = "1m";
+      OnCalendar = "daily";
+    };
+
+    inherit backupPrepareCommand;
+  };
+
+  services.postgresqlBackup = {
+    enable = true;
+    backupAll = false;
+    databases = ["piped"];
+    compression = "zstd";
+  };
+}

hosts/hetzner-vm/containers/storage/data/ports.nix

@@ -8,6 +8,7 @@
   rclone_serve_restic_vault = 4212;
   rclone_serve_restic_social = 4213;
   rclone_serve_restic_quassel = 4214;
+  rclone_serve_restic_piped = 4215;
 
   rclone_serve_http_music = 4220;
   rclone_serve_http_public = 4221;

hosts/hetzner-vm/containers/storage/profiles/rclone-serve.nix

@@ -132,6 +132,17 @@ in {
         ];
         inherit serviceConfig;
       }
+      {
+        user = "storage";
+        remote = "StorageBox:Backups/Restic/Piped";
+        type = "restic";
+        extraArgs = [
+          "--addr=0.0.0.0:${toString ports.rclone_serve_restic_piped}"
+          "--htpasswd=${secrets.restic_piped_htpasswd.path}"
+          "--baseurl=/Piped/"
+        ];
+        inherit serviceConfig;
+      }
     ];
   };
 }

hosts/hetzner-vm/containers/storage/profiles/secrets.nix

@@ -80,6 +80,7 @@
           htpasswd -bc "$secretFile" "$username" "$password" 2>/dev/null
         '';
       };
+
       restic_quassel_htpasswd = {
         user = "storage";
         group = "storage";
@@ -90,6 +91,16 @@
         '';
       };
 
+      restic_piped_htpasswd = {
+        user = "storage";
+        group = "storage";
+        fetchScript = ''
+          username=$(simple_get "/api-keys/storage/restic/Piped" .username)
+          password=$(simple_get "/api-keys/storage/restic/Piped" .password)
+          htpasswd -bc "$secretFile" "$username" "$password" 2>/dev/null
+        '';
+      };
+
       webdav_main_htpasswd = {
         user = "storage";
         group = "storage";

hosts/hetzner-vm/containers/storage/storage.nix

@@ -100,6 +100,7 @@ in {
       "/Vault/".proxyPass = "http://${containerIP}:${toString ports.rclone_serve_restic_vault}";
       "/Social/".proxyPass = "http://${containerIP}:${toString ports.rclone_serve_restic_social}";
       "/Quassel/".proxyPass = "http://${containerIP}:${toString ports.rclone_serve_restic_quassel}";
+      "/Piped/".proxyPass = "http://${containerIP}:${toString ports.rclone_serve_restic_piped}";
     };
   };
 }

hosts/hetzner-vm/data/container-addresses.nix

@@ -5,5 +5,6 @@
     social = "192.168.100.12";
     music = "192.168.100.13";
     quassel = "192.168.100.14";
+    piped = "192.168.100.15";
   };
 }

hosts/hetzner-vm/hetzner-vm.nix

@@ -9,7 +9,6 @@
     users.root
 
     hosts.hetzner-vm.modules.mailserver
-    hosts.hetzner-vm.modules.piped
 
     profiles.base
     profiles.sshd
@@ -20,9 +19,9 @@
     ./containers/social/social.nix
     ./containers/music/music.nix
     ./containers/quassel/quassel.nix
+    ./containers/piped/piped.nix
 
     hosts.hetzner-vm.profiles.restic
-    hosts.hetzner-vm.profiles.piped
     hosts.hetzner-vm.profiles.mailserver
     hosts.hetzner-vm.profiles.gitlab-static-sites
     hosts.hetzner-vm.profiles.wireguard

hosts/hetzner-vm/modules/piped/backend.nix

@@ -116,7 +116,7 @@ in {
       ensureDatabases = ["piped"];
     };
 
-    services.nginx.virtualHosts."${cfg.backendDomain}" = {
+    services.nginx.virtualHosts."${cfg.backendDomain}" = lib.mkIf (!cfg.disableNginx) {
       forceSSL = true;
       enableACME = true;
       locations."/" = {

hosts/hetzner-vm/modules/piped/default.nix

@@ -18,6 +18,12 @@ in {
     proxyDomain = mkOption {type = types.str;};
     #rydProxyDomain = mkOption { type = types.str; };
 
+    disableNginx = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Don't run nginx to serve frontend/backend/proxy";
+    };
+
     disableFrontend = mkOption {
       type = types.bool;
       default = false;
@@ -42,6 +48,30 @@ in {
       description = "Only use IPv4 querying youtube's servers for proxy";
     };
 
+    proxyNginxExtraConfig = mkOption {
+      type = types.lines;
+      default = ''
+        proxy_buffering on;
+        proxy_buffers 1024 16k;
+        proxy_set_header X-Forwarded-For "";
+        proxy_set_header CF-Connecting-IP "";
+        proxy_hide_header "alt-svc";
+        sendfile on;
+        sendfile_max_chunk 512k;
+        tcp_nopush on;
+        aio threads=default;
+        aio_write on;
+        directio 16m;
+        proxy_hide_header Cache-Control;
+        proxy_hide_header etag;
+        proxy_http_version 1.1;
+        proxy_set_header Connection keep-alive;
+        proxy_max_temp_file_size 32m;
+        access_log off;
+      '';
+      description = "Extra config for nginx on piped-proxy";
+    };
+
     httpWorkers = mkOption {
       type = types.number;
       default = 2;

hosts/hetzner-vm/modules/piped/frontend.nix

@@ -9,7 +9,7 @@ with lib; let
   frontend-package =
     pkgs.piped-frontend.override {backendDomain = cfg.backendDomain;};
 in {
-  config = mkIf (cfg.enable && !cfg.disableFrontend) {
+  config = mkIf (cfg.enable && !cfg.disableFrontend && !cfg.disableNginx) {
     services.nginx.virtualHosts."${cfg.frontendDomain}" = {
       forceSSL = true;
       enableACME = true;

hosts/hetzner-vm/modules/piped/proxy.nix

@@ -6,26 +6,6 @@
 }:
 with lib; let
   cfg = config.services.piped;
-
-  proxy_nginx_extras = ''
-    proxy_buffering on;
-    proxy_buffers 1024 16k;
-    proxy_set_header X-Forwarded-For "";
-    proxy_set_header CF-Connecting-IP "";
-    proxy_hide_header "alt-svc";
-    sendfile on;
-    sendfile_max_chunk 512k;
-    tcp_nopush on;
-    aio threads=default;
-    aio_write on;
-    directio 16m;
-    proxy_hide_header Cache-Control;
-    proxy_hide_header etag;
-    proxy_http_version 1.1;
-    proxy_set_header Connection keep-alive;
-    proxy_max_temp_file_size 32m;
-    access_log off;
-  '';
 in {
   config = mkIf (cfg.enable && !cfg.disableProxy) {
     systemd.services.piped-proxy = {
@@ -51,13 +31,13 @@ in {
       };
     };
 
-    services.nginx.virtualHosts."${cfg.proxyDomain}" = {
+    services.nginx.virtualHosts."${cfg.proxyDomain}" = lib.mkIf (!cfg.disableNginx) {
       forceSSL = true;
       enableACME = true;
       locations."/" = {
         proxyPass = "http://localhost:${toString cfg.internalProxyPort}";
         extraConfig =
-          proxy_nginx_extras
+          cfg.proxyNginxExtraConfig
           + ''
             add_header Cache-Control "public, max-age=604800";
           '';
@@ -65,7 +45,7 @@ in {
       locations."~ (/videoplayback|/api/v4/|/api/manifest/)" = {
         proxyPass = "http://localhost:${toString cfg.internalProxyPort}";
         extraConfig =
-          proxy_nginx_extras
+          cfg.proxyNginxExtraConfig
           + ''
             add_header Cache-Control private always;
           '';

hosts/hetzner-vm/secrets.nix

@@ -101,6 +101,19 @@
         '';
       };
 
+      piped_restic_password = {
+        fetchScript = ''
+          simple_get "/private-public-keys/restic/Piped" .password > $secretFile
+        '';
+      };
+      piped_restic_env = {
+        fetchScript = ''
+          RESTIC_USERNAME=$(simple_get "/api-keys/storage/restic/Piped" .username)
+          RESTIC_PASSWORD=$(simple_get "/api-keys/storage/restic/Piped" .password)
+          echo "RESTIC_REPOSITORY=rest:https://$RESTIC_USERNAME:$RESTIC_PASSWORD@storage-restic.owo.monster/Piped" > $secretFile
+        '';
+      };
+
       restic_password = {
         fetchScript = ''
           simple_get "/private-public-keys/restic/HetznerVM" .password > $secretFile

treeConfig.nix

@@ -16,7 +16,8 @@
     "hosts/hetzner-vm/modules/piped".functor.enable = true;
     "hosts/hetzner-vm/containers/storage/profiles".functor.enable = true;
     "hosts/hetzner-vm/containers/social/profiles".functor.enable = true;
-    "hosts/hetzner-vm/containers/irc/profiles".functor.enable = true;
+    "hosts/hetzner-vm/containers/quassel/profiles".functor.enable = true;
+    "hosts/hetzner-vm/containers/piped/profiles".functor.enable = true;
 
     # Profiles
     "profiles/*".functor.enable = true;