restore shenanigan

deployNodes.nix

@@ -9,7 +9,7 @@ in {
     };
   };
   hetzner-vm = {
-    hostname = "hetzner-vm.tailscale-internal.genderfucked.monster";
+    hostname = "hetzner-vm.servers.genderfucked.monster";
     username = "root";
     profiles.system = {
       user = "root";

guides/HETZNERVM_RESTORE_GUIDE.md

@@ -0,0 +1,73 @@
+# ok so you fucked up, what next,
+## Step 1
+Either create a new VM or if still have access, go to ISOs and put in nixos 22.05 iso and reboot server.
+If create new vm, please change DNS and update IPs in the main nix config and this document (v6) and set rhost
+## Step 2 
+Use console to set ssh password.
+
+SSH in `ssh root@hetzner-vm.servers.genderfucked.monster`
+
+Add ssh key `echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAeN3T1aZkTm5xS0b66cRDyKUbdEQCFyzVWXeW+eIbsa chaos@chaos" >> /root/.ssh/authorized_keys`
+## Step 3
+Install NixOS.
+```
+parted -s "/dev/sda" -- mklabel msdos
+parted -s "/dev/sda" -- mkpart primary 512MiB 100%
+parted -s "/dev/sda" -- print
+
+mkfs.ext4 /dev/sda1
+
+mount /dev/sda1 /mnt
+
+nixos-generate-config --root /mnt
+```
+
+Edit configuration.nix and replace with
+```nix
+{ config, pkgs, ... }:
+{
+  imports = [ ./hardware-configuration.nix ];
+  users.users.root = {
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAeN3T1aZkTm5xS0b66cRDyKUbdEQCFyzVWXeW+eIbsa chaos@chaos"
+    ];
+  };
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/sda";
+  boot.initrd.kernelModules = [ "nvme" ];
+
+  services.openssh.enable = true;
+
+  networking.enableIPv6 = true;
+  networking.usePredictableInterfaceNames = false;
+  networking.dhcpcd.enable = true;
+  systemd.network = {
+    enable = true;
+    networks.eth0 = {
+      name = "eth0";
+      address = [ "2a01:4f9:c010:8beb::1/64" ];
+      gateway = [ "fe80::1" ];
+    };
+  };
+
+  system.stateVersion = "21.11"; 
+}
+```
+
+Then run `nixos-install` and reboot if successful.
+
+## Step 4
+Redeploy with `deploy -s .#hetzner-vm --magic-rollback=false --auto-rollback=false --ssh-user root`
+
+## Step 5 
+Restore using restic.
+`B2_ACCOUNT_ID=id B2_ACCOUNT_KEY=key RESTIC_REPOSITORY=b2:Chaos-Backups:HetznerVM restic restore latest`
+
+## Step 6
+re-connect to tailscale `tailscale up` and redo dns
+
+## Step 7
+Restore Postgres `cat /var/backup/postgresql/all.sql.zstd | zstd -d | sudo -u postgres psql`
+
+## Step 8 
+Restore music, either via `mkdir -p /storage/music && env B2_ACCOUNT_ID=id B2_ACCOUNT_KEY=key RESTIC_REPOSITORY=b2:Chaos-Backups:Music restic restore latest --target /storage/music` or ./sync.sh on laptop.

hosts/hetzner-vm/services/restic.nix

@@ -1,19 +1,32 @@
-{ lib, pkgs, ... }:
+{ lib, config, pkgs, ... }:
 let
 
   mail_config = (import ./mailserver/config.nix { });
 
   backupUser = "root";
   backupPaths = [
-    "/var/lib/postgresql"
+    "/secrets"
+
     "/var/lib/vault"
     "/var/lib/acme"
-    "/secrets"
+    # Quassel & Invidious
+    "/var/backup/postgresql"
+    "/home/quassel/.config/quassel-irc.org"
+    # MPD State
+    "/mpd"
+    # doesn't work for restoring might as well not backup
+    # "/var/lib/tailscale"
+
+    # mail
     mail_config.vmail_config.directory
     mail_config.sieve_directory
     mail_config.dkim_directory
     "/var/lib/redis-rspamd"
   ];
+  backupPrepareCommand = "${(pkgs.writeShellScriptBin "backupPrepareCommand" ''
+    systemctl start postgresqlBackup --wait
+  '')}/bin/backupPrepareCommand";
+
   timerConfig = {
     OnBootSec = "1m";
     OnCalendar = "daily";
@@ -31,16 +44,16 @@ let
     };
   };
 
-  restic_commands = lib.mapAttrsToList (repoName: repoInfo: (
+  restic_commands = lib.mapAttrsToList (repoName: repoInfo:
-    # nya
+    (
-    pkgs.writeShellScriptBin "restic-${repoName}" ''
+      # nya
-      env \
+      pkgs.writeShellScriptBin "restic-${repoName}" ''
-        $(cat ${repoInfo.environmentFile}) \
+        env \
-        RESTIC_PASSWORD_FILE=${repoInfo.passwordFile} \
+          $(cat ${repoInfo.environmentFile}) \
-        RESTIC_REPOSITORY=${repoInfo.repository} \
+          RESTIC_PASSWORD_FILE=${repoInfo.passwordFile} \
-      ${pkgs.restic}/bin/restic $@
+          RESTIC_REPOSITORY=${repoInfo.repository} \
-    ''
+        ${pkgs.restic}/bin/restic $@
-  )) repos;
+      '')) repos;
 in {
   environment.systemPackages = restic_commands;
 
@@ -49,6 +62,7 @@ in {
       user = backupUser;
       paths = backupPaths;
       inherit timerConfig;
+      inherit backupPrepareCommand;
     }
     repos.Chaos-Backups-HetznerVM
   ];
@@ -57,7 +71,14 @@ in {
       user = backupUser;
       paths = backupPaths;
       inherit timerConfig;
+      inherit backupPrepareCommand;
     }
     repos.Cassie-Backups-HetznerVM
   ];
+
+  services.postgresqlBackup = {
+    enable = true;
+    backupAll = true;
+    compression = "zstd";
+  };
 }