From ffd17fe1238451622eb26d1753f6536b1ce0255f Mon Sep 17 00:00:00 2001 From: Chaos Date: Thu, 27 Oct 2022 20:27:22 +0100 Subject: [PATCH] beeppppppppp --- .gitignore | 1 - deployNodes.nix | 2 + flake.lock | 18 +++--- home/apps/musicutil.nix | 4 +- home/dev/all/extra.nix | 1 + home/dev/all/git.nix | 2 +- hosts/hetzner-vm/services/lappy-dev.nix | 5 +- hosts/nixos.nix | 2 +- hosts/storage/hardware.nix | 6 +- hosts/storage/misc.nix | 3 +- hosts/storage/modules/rclone-serve.nix | 62 ++++++++++++++++++ hosts/storage/networking.nix | 19 ++++++ hosts/storage/populate-rclone-config.sh | 39 ++++++++++++ hosts/storage/rclone_config.template | 19 ++++++ hosts/storage/storage.nix | 85 +++++++++++++++++++++++-- 15 files changed, 244 insertions(+), 24 deletions(-) create mode 100644 hosts/storage/modules/rclone-serve.nix create mode 100644 hosts/storage/networking.nix create mode 100755 hosts/storage/populate-rclone-config.sh diff --git a/.gitignore b/.gitignore index 80ce9c0..1b0a8f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -networking.nix result *.qcow2 \ No newline at end of file diff --git a/deployNodes.nix b/deployNodes.nix index 8ec2c6e..cc92b52 100644 --- a/deployNodes.nix +++ b/deployNodes.nix @@ -13,6 +13,7 @@ in { username = "root"; profiles.system = { user = "root"; + sshUser = "root"; path = activateNixOS_x64_64-linux nixosConfigurations.hetzner-vm; }; }; @@ -21,6 +22,7 @@ in { username = "root"; profiles.system = { user = "root"; + sshUser = "root"; path = activateNixOS_x64_64-linux nixosConfigurations.storage; }; }; diff --git a/flake.lock b/flake.lock index 5d75a23..1ffadca 100644 --- a/flake.lock +++ b/flake.lock @@ -96,11 +96,11 @@ "utils": "utils_3" }, "locked": { - "lastModified": 1666463764, - "narHash": "sha256-NmayV9S0s7CgNEA2QbIxDU0VCIiX6bIHu8PCQPnYHDM=", + "lastModified": 1666875108, + "narHash": "sha256-sf0uvlDIatV/eYUJ8N5+Si21og3B6G+AKXive3RUH4E=", "owner": "nix-community", "repo": "home-manager", - "rev": "69d19b9839638fc487b370e0600a03577a559081", + "rev": "32fe7d2ebb7e338ad95a3ea9393fc6ad681368ce", "type": "github" }, "original": { @@ -138,11 +138,11 @@ ] }, "locked": { - "lastModified": 1665392861, - "narHash": "sha256-bCd8fYJMAb0LzabsiXl4nxECDoz483bJOCa2hjox7N0=", + "lastModified": 1666776005, + "narHash": "sha256-HwSMF19PpczfqNHKcFsA6cF4PVbG00uUSdbq6q3jB5o=", "owner": "lnl7", "repo": "nix-darwin", - "rev": "ef56fd8979b5f4e800c4716f62076e00600b1172", + "rev": "f6648ca0698d1611d7eadfa72b122252b833f86c", "type": "github" }, "original": { @@ -186,11 +186,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1666377499, - "narHash": "sha256-dZZCGvWcxc7oGnUgFVf0UeNHsJ4VhkTM0v5JRe8EwR8=", + "lastModified": 1666703756, + "narHash": "sha256-GwpMJ1hT+z1fMAUkaGtvbvofJQwdVFDEGVhfE82+AUk=", "owner": "nixos", "repo": "nixpkgs", - "rev": "301aada7a64812853f2e2634a530ef5d34505048", + "rev": "f994293d1eb8812f032e8919e10a594567cf6ef7", "type": "github" }, "original": { diff --git a/home/apps/musicutil.nix b/home/apps/musicutil.nix index 5ffad63..835e1c5 100644 --- a/home/apps/musicutil.nix +++ b/home/apps/musicutil.nix @@ -1,3 +1 @@ -{ inputs, pkgs, ... }: { - home.packages = with pkgs; [ musicutil ]; -} +{ inputs, pkgs, ... }: { home.packages = with pkgs; [ musicutil ]; } diff --git a/home/dev/all/extra.nix b/home/dev/all/extra.nix index 6932226..fe92012 100644 --- a/home/dev/all/extra.nix +++ b/home/dev/all/extra.nix @@ -7,6 +7,7 @@ tmux socat file + elvish (pkgs.busybox.override { enableAppletSymlinks = false; extraConfig = '' diff --git a/home/dev/all/git.nix b/home/dev/all/git.nix index 8a4f65b..c463a7a 100644 --- a/home/dev/all/git.nix +++ b/home/dev/all/git.nix @@ -3,7 +3,7 @@ enable = true; lfs.enable = true; package = pkgs.gitAndTools.gitFull; - userName = "ChaotiCryptidz"; + userName = "Chaos"; userEmail = "chaoticryptidz@owo.monster"; extraConfig = { credential = { helper = "store"; }; }; }; diff --git a/hosts/hetzner-vm/services/lappy-dev.nix b/hosts/hetzner-vm/services/lappy-dev.nix index 631ff0f..34077e9 100644 --- a/hosts/hetzner-vm/services/lappy-dev.nix +++ b/hosts/hetzner-vm/services/lappy-dev.nix @@ -2,6 +2,9 @@ services.nginx.virtualHosts."lappy-dev.owo.monster" = { forceSSL = true; enableACME = true; - locations = { "/".proxyPass = "http://lappy.tailscale-internal.genderfucked.monster:8088"; }; + locations = { + "/".proxyPass = + "http://lappy.tailscale-internal.genderfucked.monster:8088"; + }; }; } diff --git a/hosts/nixos.nix b/hosts/nixos.nix index d397531..648d3a3 100644 --- a/hosts/nixos.nix +++ b/hosts/nixos.nix @@ -47,7 +47,7 @@ in { storage = nixosUnstableSystem { specialArgs = defaultSpecialArgs; system = "x86_64-linux"; - modules = defaultModules ++ [ ./storage/storage.nix ]; + modules = defaultModules ++ [ ./storage/modules/rclone-serve.nix ./storage/storage.nix ]; }; # nix --no-sandbox build .#nixosConfigurations.raspberry.config.system.build.sdImage diff --git a/hosts/storage/hardware.nix b/hosts/storage/hardware.nix index 047e427..8ee92b3 100644 --- a/hosts/storage/hardware.nix +++ b/hosts/storage/hardware.nix @@ -1,5 +1,7 @@ -{ ...}: { +{ modulesPath, ... }: { imports = [ + (modulesPath + "/profiles/qemu-guest.nix") + ]; boot.loader.grub.enable = true; @@ -9,4 +11,4 @@ device = "/dev/sda1"; fsType = "ext4"; }; -} \ No newline at end of file +} diff --git a/hosts/storage/misc.nix b/hosts/storage/misc.nix index 49d0dea..df7c466 100644 --- a/hosts/storage/misc.nix +++ b/hosts/storage/misc.nix @@ -1,9 +1,8 @@ -{...}: { +{ ... }: { nix.settings.auto-optimise-store = true; nix.gc = { automatic = true; dates = "daily"; options = "--delete-older-than 1d"; }; - } diff --git a/hosts/storage/modules/rclone-serve.nix b/hosts/storage/modules/rclone-serve.nix new file mode 100644 index 0000000..bee212b --- /dev/null +++ b/hosts/storage/modules/rclone-serve.nix @@ -0,0 +1,62 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.rclone-serve; + + makeNameSafe = name: builtins.replaceStrings [ "/" ] [ "-" ] name; + + daemonService = serve_config: { + enable = true; + requires = [ "network.target" ]; + after = [ "network.target" ] + ++ (if serve_config.after != null then serve_config.after else [ ]); + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "simple"; + Restart = "on-failure"; + RestartSec = "5s"; + + User = + if serve_config.user != null then "${serve_config.user}" else "root"; + + ExecStart = + "${pkgs.rclone}/bin/rclone serve ${serve_config.type} ${serve_config.remote} ${ + lib.concatStrings serve_config.extraArgs + }"; + }; + }; +in { + options = { + services.rclone-serve = { + enable = mkOption { + type = types.bool; + default = false; + }; + + remotes = mkOption { + type = types.listOf (types.submodule { + options = { + remote = mkOption { type = types.str; }; + type = mkOption { type = types.str; }; + user = mkOption { type = types.str; }; + after = mkOption { type = types.listOf types.str; }; + + extraArgs = mkOption { type = types.listOf types.str; }; + }; + }); + default = [ ]; + }; + }; + }; + + config = mkMerge [ + (mkIf (cfg.enable && cfg.remotes != [ ]) { + systemd.services = listToAttrs (map (remote: { + name = "rclone-serve-${makeNameSafe remote.remote}-${ + makeNameSafe remote.type + }"; + value = daemonService remote; + }) cfg.remotes); + }) + ]; +} diff --git a/hosts/storage/networking.nix b/hosts/storage/networking.nix new file mode 100644 index 0000000..95e0bb3 --- /dev/null +++ b/hosts/storage/networking.nix @@ -0,0 +1,19 @@ +{ ... }: { + systemd.services.systemd-networkd-wait-online.enable = false; + + networking.firewall.enable = true; + networking.firewall.allowPing = true; + networking.firewall.allowedTCPPorts = [ 22 ]; + + networking.enableIPv6 = true; + networking.usePredictableInterfaceNames = false; + networking.dhcpcd.enable = true; + systemd.network = { + enable = true; + networks.eth0 = { + name = "eth0"; + address = [ "2a01:4f9:c010:3e92::1/64" ]; + gateway = [ "fe80::1" ]; + }; + }; +} diff --git a/hosts/storage/populate-rclone-config.sh b/hosts/storage/populate-rclone-config.sh new file mode 100755 index 0000000..254a7fe --- /dev/null +++ b/hosts/storage/populate-rclone-config.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -ex + +kv_get() { + vault kv get -format json ${1} +} + +simple_get() { + kv_get ${1} | jq .data.data${2} -r +} + +simple_get_obscure() { + rclone obscure $(simple_get $@) +} + +VAULT_USERNAME=$1 +VAULT_PASSWORD_FILE=$2 +TEMPLATE_FILE=$3 +OUTPUT_FILE=$4 + +vault login -no-print -method=userpass username=${VAULT_USERNAME} password=$(cat ${VAULT_PASSWORD_FILE}) + +TMP_DIR="$(mktemp -d)" + +cp ${TEMPLATE_FILE} "${TMP_DIR}/template" + +pushd "${TMP_DIR}" +STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password) +sed -i "s/STORAGEBOX_PASSWORD/${STORAGEBOX_PASSWORD}/" ./template + +B2_CHAOS_BACKUPS_ACCOUNT=$(simple_get /api-keys/backblaze/Chaos-Backups .keyID) +B2_CHAOS_BACKUPS_KEY=$(simple_get /api-keys/backblaze/Chaos-Backups .applicationKey) +sed -i "s/B2_CHAOS_BACKUPS_ACCOUNT/${B2_CHAOS_BACKUPS_ACCOUNT}/" ./template +sed -i "s/B2_CHAOS_BACKUPS_KEY/${B2_CHAOS_BACKUPS_KEY}/" ./template +popd + +cat "${TMP_DIR}/template" > "${OUTPUT_FILE}" +rm -rf "${TMP_DIR}" \ No newline at end of file diff --git a/hosts/storage/rclone_config.template b/hosts/storage/rclone_config.template index e69de29..7dfe73e 100644 --- a/hosts/storage/rclone_config.template +++ b/hosts/storage/rclone_config.template @@ -0,0 +1,19 @@ +[StorageBox-Remote] +type = smb +host = u323231.your-storagebox.de +user = u323231 +pass = STORAGEBOX_PASSWORD + +[StorageBox] +type = alias +remote = StorageBox-Remote:backup + +[B2-Chaos-Backups-Source] +type = b2 +account = B2_CHAOS_BACKUPS_ACCOUNT +key = B2_CHAOS_BACKUPS_KEY +hard_delete = true + +[B2-Chaos-Backups] +type = alias +remote = B2-Chaos-Backups-Source:Chaos-Backups \ No newline at end of file diff --git a/hosts/storage/storage.nix b/hosts/storage/storage.nix index bd42341..c97b49d 100644 --- a/hosts/storage/storage.nix +++ b/hosts/storage/storage.nix @@ -9,17 +9,94 @@ profiles.tailscale profiles.sshd - ./storage.nix ./hardware.nix ./misc.nix - (modulesPath + "/profiles/qemu-guest.nix") - ../../extras/laura-ssh-root.nix ]; - environment.systemPackages = with pkgs; [ rclone cifs-utils ]; + users.groups.storage = { }; + users.users.storage = { + isNormalUser = true; + extraGroups = [ "storage" ]; + }; + systemd.services.populate-rclone-config = { + wantedBy = [ "multi-user.target" ]; + requires = [ "network.target" ]; + after = [ "network.target" ]; + path = with pkgs; [ bash rclone vault getent jq ]; + script = let + vault_username = "storage"; + vault_password_file = "/secrets/vault_password"; + in '' + mkdir -p /home/storage/.config/rclone + + VAULT_ADDR="https://vault.owo.monster" bash ${ + ./populate-rclone-config.sh + } ${vault_username} ${vault_password_file} ${ + ./rclone_config.template + } /home/storage/.config/rclone/rclone.conf + chown storage:storage /home/storage/.config/rclone/rclone.conf + chmod 660 /home/storage/.config/rclone/rclone.conf + ''; + }; + + systemd.tmpfiles.rules = [ "d /storage 0755 storage storage -" ]; + systemd.services.storage-mount = { + wantedBy = [ "multi-user.target" ]; + requires = [ + "network.target" + "populate-rclone-config.service" + "systemd-tmpfiles-setup.service" + ]; + after = [ + "network.target" + "populate-rclone-config.service" + "systemd-tmpfiles-setup.service" + ]; + path = with pkgs; [ bash rclone mount ]; + script = '' + set -e + umount /storage || true + rclone --config /home/storage/.config/rclone/rclone.conf mount StorageBox: /storage + ''; + }; + + security.acme = { + defaults = { email = "chaoticryptidz@owo.monster"; }; + acceptTerms = true; + }; + services.nginx = { + enable = true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + commonHttpConfig = ""; + clientMaxBodySize = "512m"; + serverNamesHashBucketSize = 1024; + }; + networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.rclone-serve = { + enable = true; + remotes = [{ + user = "storage"; + remote = "StorageBox:Chaos-Backups/DNS"; + type = "webdav"; + after = [ "populate-rclone-config.service" ]; + extraArgs = [ "--addr=:4242" ]; + }]; + }; + + services.nginx.virtualHosts."storage-web.owo.monster" = { + forceSSL = true; + enableACME = true; + #locations = { "/".proxyPass = "http://localhost:4242"; }; + }; + + environment.systemPackages = with pkgs; [ rclone cifs-utils ]; home-manager.users.root = { imports = with tree; [ home.base home.dev.small ];