From d69991819ab980ef0350aad4fd93c50a4a7e742a Mon Sep 17 00:00:00 2001 From: chaos Date: Thu, 14 Sep 2023 19:44:27 +0100 Subject: [PATCH] piped-proxy on raspberry --- data/chaos_wireguard_internal.nix | 1 + flake.lock | 6 +-- home/base/ssh.nix | 2 +- hosts/hetzner-vm/containers/piped/default.nix | 10 +++- .../containers/piped/profiles/piped.nix | 8 ++-- .../hetzner-vm/containers/storage/default.nix | 3 +- .../{profiles => }/rclone_config.template | 0 .../storage/{profiles => }/secrets.nix | 0 hosts/hetzner-vm/hetzner-vm.nix | 2 + hosts/hetzner-vm/profiles/wireguard.nix | 2 + hosts/lappy-t495/lappy-t495.nix | 1 + .../profiles/auto-storage-backups.nix | 2 + hosts/raspberry/profiles/external-drive.nix | 8 ++-- hosts/raspberry/profiles/piped-proxy.nix | 25 ++++++++++ hosts/raspberry/profiles/wireguard.nix | 1 + hosts/raspberry/raspberry.nix | 5 ++ hosts/raspberry/rclone_config.template | 17 +++++++ hosts/raspberry/secrets.nix | 37 +++++++++++++- hosts/vault/profiles/wireguard.nix | 2 + hosts/vault/vault.nix | 1 + outputs.nix | 5 ++ profiles/nginx-firewall.nix | 4 ++ profiles/remote-builders.nix | 48 +++++++++++++++++++ scripts/deploy-all.sh | 12 ++--- scripts/deploy/hetzner-vm.sh | 9 ++++ scripts/deploy/raspberry.sh | 9 ++++ scripts/deploy/vault.sh | 9 ++++ 27 files changed, 208 insertions(+), 21 deletions(-) rename hosts/hetzner-vm/containers/storage/{profiles => }/rclone_config.template (100%) rename hosts/hetzner-vm/containers/storage/{profiles => }/secrets.nix (100%) create mode 100644 hosts/raspberry/profiles/auto-storage-backups.nix create mode 100644 hosts/raspberry/profiles/piped-proxy.nix create mode 100644 hosts/raspberry/rclone_config.template create mode 100644 profiles/nginx-firewall.nix create mode 100644 profiles/remote-builders.nix create mode 100755 scripts/deploy/hetzner-vm.sh create mode 100644 scripts/deploy/raspberry.sh create mode 100644 scripts/deploy/vault.sh diff --git a/data/chaos_wireguard_internal.nix b/data/chaos_wireguard_internal.nix index 77fbd85..a028137 100644 --- a/data/chaos_wireguard_internal.nix +++ b/data/chaos_wireguard_internal.nix @@ -23,6 +23,7 @@ raspberry = { ip = "10.69.42.5"; public = "IGq+WanFM/bKNUkwjO/0AAtDhJLvtvU+mVxH27QyHTc="; + endpoint = "raspberry.servers.genderfucked.monster:51820"; }; }; } diff --git a/flake.lock b/flake.lock index b17027f..00a744f 100644 --- a/flake.lock +++ b/flake.lock @@ -202,11 +202,11 @@ ] }, "locked": { - "lastModified": 1694550850, - "narHash": "sha256-lN3W5WhbxLrsp2lgGh4fk2th5RkhQIMFd4WCNAlYvs8=", + "lastModified": 1694714112, + "narHash": "sha256-x2CgsEvWIf4ETx4puLqmHrPthesNXW30OMYg4pVJARg=", "owner": "ChaotiCryptidz", "repo": "piped-flake", - "rev": "6ef001f2a840fe3fefbcda619126659ba9cc1981", + "rev": "c5ad01eac79809686d9685671e77a6dac4e17ddf", "type": "gitlab" }, "original": { diff --git a/home/base/ssh.nix b/home/base/ssh.nix index 0b321a1..9dcc467 100644 --- a/home/base/ssh.nix +++ b/home/base/ssh.nix @@ -4,7 +4,7 @@ in { programs.ssh.enable = true; programs.ssh.matchBlocks = lib.mkMerge - ((lib.forEach ["hetzner-vm" "vault" "vault-decrypt"] (hostname: { + ((lib.forEach ["hetzner-vm" "vault" "raspberry" "vault-decrypt"] (hostname: { "${hostname}" = { user = "root"; hostname = "${hostname}.servers.genderfucked.monster"; diff --git a/hosts/hetzner-vm/containers/piped/default.nix b/hosts/hetzner-vm/containers/piped/default.nix index fe3d66e..eed90e7 100644 --- a/hosts/hetzner-vm/containers/piped/default.nix +++ b/hosts/hetzner-vm/containers/piped/default.nix @@ -5,6 +5,8 @@ config, ... }: let + internal_wireguard = import ../../../../data/chaos_wireguard_internal.nix {}; + container-addresses = import ../../data/container-addresses.nix {}; hostIP = container-addresses.host; containerIP = container-addresses.containers.piped; @@ -104,8 +106,12 @@ in { forceSSL = true; enableACME = true; locations."/" = { - proxyPass = "http://unix:${pipedSocketForComponent "proxy"}"; - extraConfig = config.services.piped.proxyNginxExtraConfig; + proxyPass = "http://${internal_wireguard.hosts.raspberry.ip}"; + extraConfig = '' + proxy_set_header Host $host; + ''; + #proxyPass = "http://unix:${pipedSocketForComponent "proxy"}"; + #extraConfig = config.services.piped.proxyNginxExtraConfig; }; }; diff --git a/hosts/hetzner-vm/containers/piped/profiles/piped.nix b/hosts/hetzner-vm/containers/piped/profiles/piped.nix index 470bda5..7a2e1a2 100644 --- a/hosts/hetzner-vm/containers/piped/profiles/piped.nix +++ b/hosts/hetzner-vm/containers/piped/profiles/piped.nix @@ -8,6 +8,8 @@ in { backendDomain = "backend.piped.owo.monster"; proxyDomain = "proxy.piped.owo.monster"; + disableProxy = true; # Currently using proxy via raspberry in UK + nginxForceSSL = false; nginxEnableACME = false; @@ -31,7 +33,7 @@ in { config.services.nginx.virtualHosts."${piped_config.backendDomain}" = { extraConfig = "listen unix:/var/sockets/piped-backend.sock;"; }; - config.services.nginx.virtualHosts."${piped_config.proxyDomain}" = { - extraConfig = "listen unix:/var/sockets/piped-proxy.sock;"; - }; + #config.services.nginx.virtualHosts."${piped_config.proxyDomain}" = { + # extraConfig = "listen unix:/var/sockets/piped-proxy.sock;"; + #}; } diff --git a/hosts/hetzner-vm/containers/storage/default.nix b/hosts/hetzner-vm/containers/storage/default.nix index a796e23..934b7d3 100644 --- a/hosts/hetzner-vm/containers/storage/default.nix +++ b/hosts/hetzner-vm/containers/storage/default.nix @@ -41,10 +41,11 @@ in { modules.nixos.rclone-sync modules.nixos.secrets + ./secrets.nix + users.root ] ++ (with hosts.hetzner-vm.containers.storage; [ - profiles.secrets profiles.auto-secrets profiles.rclone-configs profiles.rclone-serve diff --git a/hosts/hetzner-vm/containers/storage/profiles/rclone_config.template b/hosts/hetzner-vm/containers/storage/rclone_config.template similarity index 100% rename from hosts/hetzner-vm/containers/storage/profiles/rclone_config.template rename to hosts/hetzner-vm/containers/storage/rclone_config.template diff --git a/hosts/hetzner-vm/containers/storage/profiles/secrets.nix b/hosts/hetzner-vm/containers/storage/secrets.nix similarity index 100% rename from hosts/hetzner-vm/containers/storage/profiles/secrets.nix rename to hosts/hetzner-vm/containers/storage/secrets.nix diff --git a/hosts/hetzner-vm/hetzner-vm.nix b/hosts/hetzner-vm/hetzner-vm.nix index 8d6d57f..0ebc5d9 100644 --- a/hosts/hetzner-vm/hetzner-vm.nix +++ b/hosts/hetzner-vm/hetzner-vm.nix @@ -12,8 +12,10 @@ profiles.base profiles.sshd profiles.nginx + profiles.nginx-firewall profiles.nix-gc profiles.kernels.latest + profiles.cross.arm64 ./networking.nix ./hardware.nix diff --git a/hosts/hetzner-vm/profiles/wireguard.nix b/hosts/hetzner-vm/profiles/wireguard.nix index 62e6d2d..3b515ac 100644 --- a/hosts/hetzner-vm/profiles/wireguard.nix +++ b/hosts/hetzner-vm/profiles/wireguard.nix @@ -33,6 +33,8 @@ in { publicKey = "${data.hosts.raspberry.public}"; presharedKeyFile = "${secrets.wg_preshared_raspberry.path}"; allowedIPs = ["${data.hosts.raspberry.ip}/32"]; + endpoint = "${data.hosts.raspberry.endpoint}"; + persistentKeepalive = 25; } ]; }; diff --git a/hosts/lappy-t495/lappy-t495.nix b/hosts/lappy-t495/lappy-t495.nix index a9ef8cd..954d0d2 100644 --- a/hosts/lappy-t495/lappy-t495.nix +++ b/hosts/lappy-t495/lappy-t495.nix @@ -10,6 +10,7 @@ presets.nixos.encrypted-usb profiles.cross.arm64 + profiles.remote-builders hosts.lappy-t495.profiles.wireguard diff --git a/hosts/raspberry/profiles/auto-storage-backups.nix b/hosts/raspberry/profiles/auto-storage-backups.nix new file mode 100644 index 0000000..ea8f50d --- /dev/null +++ b/hosts/raspberry/profiles/auto-storage-backups.nix @@ -0,0 +1,2 @@ +{...}: { +} diff --git a/hosts/raspberry/profiles/external-drive.nix b/hosts/raspberry/profiles/external-drive.nix index 7fef7d9..85a1831 100644 --- a/hosts/raspberry/profiles/external-drive.nix +++ b/hosts/raspberry/profiles/external-drive.nix @@ -37,8 +37,8 @@ in { systemd.tmpfiles.rules = ["d ${external_drive_data.mountpoint} - root root"]; - # services.udev.extraRules = '' - # ACTION=="add", ENV{PARTNAME}=="${external_drive_data.encrypted_partlabel}", ENV{SYSTEMD_WANTS}="mount-external-drive.service" - # ACTION=="remove", ENV{PARTNAME}=="${external_drive_data.encrypted_partlabel}", ENV{SYSTEMD_WANTS}="unmount-external-drive.service" - # ''; + #services.udev.extraRules = '' + # ACTION=="add", ENV{PARTLABEL}=="${external_drive_data.encrypted_label}", ENV{SYSTEMD_WANTS}="mount-external-drive.service" + # ACTION=="remove", ENV{PARTLABEL}=="${external_drive_data.encrypted_label}", ENV{SYSTEMD_WANTS}="unmount-external-drive.service" + #''; } diff --git a/hosts/raspberry/profiles/piped-proxy.nix b/hosts/raspberry/profiles/piped-proxy.nix new file mode 100644 index 0000000..d274a5c --- /dev/null +++ b/hosts/raspberry/profiles/piped-proxy.nix @@ -0,0 +1,25 @@ +{ + config, + pkgs, + ... +}: { + config.services.piped = { + enable = true; + + # Takes too much time to compile otherwise, idm extra bandwidth + proxyPackage = + (pkgs.piped-proxy.override { + withAVIF = false; + withWebP = false; + }) + .overrideAttrs { + pname = "piped-proxy-debug"; + doCheck = false; + buildType = "debug"; + }; + proxyDomain = "proxy.piped.owo.monster"; + + disableBackend = true; + disableFrontend = true; + }; +} diff --git a/hosts/raspberry/profiles/wireguard.nix b/hosts/raspberry/profiles/wireguard.nix index b404d1d..33afd1e 100644 --- a/hosts/raspberry/profiles/wireguard.nix +++ b/hosts/raspberry/profiles/wireguard.nix @@ -5,6 +5,7 @@ persistentKeepalive = 15; in { networking.firewall.trustedInterfaces = ["wg0"]; + networking.firewall.allowedUDPPorts = [51820]; networking.wg-quick.interfaces = { wg0 = { address = ["${data.hosts.raspberry.ip}/32"]; diff --git a/hosts/raspberry/raspberry.nix b/hosts/raspberry/raspberry.nix index ee997a1..7a4272d 100644 --- a/hosts/raspberry/raspberry.nix +++ b/hosts/raspberry/raspberry.nix @@ -15,6 +15,8 @@ in { profiles.base profiles.sshd + profiles.nginx + profiles.nginx-firewall profiles.connectivity.network_manager profiles.connectivity.bluetooth @@ -27,6 +29,9 @@ in { ] ++ (with hosts.raspberry.profiles; [ external-drive + wireguard + piped-proxy + auto-storage-backups ]); environment.systemPackages = [ diff --git a/hosts/raspberry/rclone_config.template b/hosts/raspberry/rclone_config.template new file mode 100644 index 0000000..5c140df --- /dev/null +++ b/hosts/raspberry/rclone_config.template @@ -0,0 +1,17 @@ +[StorageBox-Remote] +type = webdav +vendor = other +host = u323231.your-storagebox.de +url = https://u323231.your-storagebox.de +user = u323231 +pass = STORAGEBOX_PASSWORD + +[StorageBox-Hasher] +type = hasher +remote = StorageBox-Remote: +hashes = sha1,md5 +max_age = off + +[StorageBox] +type = alias +remote = StorageBox-Hasher: \ No newline at end of file diff --git a/hosts/raspberry/secrets.nix b/hosts/raspberry/secrets.nix index 9b6d932..fab461f 100644 --- a/hosts/raspberry/secrets.nix +++ b/hosts/raspberry/secrets.nix @@ -1,6 +1,22 @@ -{...}: { +{pkgs, ...}: { services.secrets = { enable = true; + + packages = with pkgs; [rclone]; + + extraFunctions = '' + simple_get_obscure() { + rclone obscure "$(simple_get "$@")" + } + ''; + + requiredVaultPaths = [ + "private-public-keys/data/cryptsetup/raspberry-ext-drive" # used dynamically + "private-public-keys/data/wireguard/chaos-internal/raspberry" + "passwords/data/wifi/parentals-home" + "api-keys/data/hetzner/storagebox" + ]; + secrets = { # Used for fetching the encryption drive's key at runtime # can be revoked in case of hardware theft @@ -31,6 +47,25 @@ ''; }; + rclone_config = { + fetchScript = '' + TMP_DIR="$(mktemp -d)" + + cp ${./rclone_config.template} "$TMP_DIR/template" + + pushd "$TMP_DIR" >/dev/null + + STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password) + sed -i "s/STORAGEBOX_PASSWORD/$STORAGEBOX_PASSWORD/" ./template + + cp ./template "$secretFile" + + popd >/dev/null + + rm -rf "$TMP_DIR" + ''; + }; + # for internal wireguard VPN wg_priv = { fetchScript = '' diff --git a/hosts/vault/profiles/wireguard.nix b/hosts/vault/profiles/wireguard.nix index 4e2ae28..c829462 100644 --- a/hosts/vault/profiles/wireguard.nix +++ b/hosts/vault/profiles/wireguard.nix @@ -34,6 +34,8 @@ in { publicKey = "${data.hosts.raspberry.public}"; presharedKeyFile = "${secrets.wg_preshared_raspberry.path}"; allowedIPs = ["${data.hosts.raspberry.ip}/32"]; + endpoint = "${data.hosts.raspberry.endpoint}"; + persistentKeepalive = 25; } ]; }; diff --git a/hosts/vault/vault.nix b/hosts/vault/vault.nix index a17b069..8a01c75 100644 --- a/hosts/vault/vault.nix +++ b/hosts/vault/vault.nix @@ -6,6 +6,7 @@ profiles.sshd profiles.nix-gc profiles.nginx + profiles.nginx-firewall profiles.kernels.latest hosts.vault.profiles.wireguard diff --git a/outputs.nix b/outputs.nix index 5db9b53..fb0f328 100644 --- a/outputs.nix +++ b/outputs.nix @@ -19,6 +19,7 @@ in system: let pkgs = import nixpkgs { inherit system; + config.allowUnfree = true; overlays = [ (import ./overlay) ]; @@ -26,6 +27,10 @@ in in lib.foldl' lib.recursiveUpdate {} [ { + # we expose nixpkgs.${system} so that we can nix run/build stuff + # from nixpkgs from flake's input versions + nixpkgs = pkgs; + formatter = pkgs.alejandra; devShell = pkgs.mkShell { diff --git a/profiles/nginx-firewall.nix b/profiles/nginx-firewall.nix new file mode 100644 index 0000000..93e302c --- /dev/null +++ b/profiles/nginx-firewall.nix @@ -0,0 +1,4 @@ +{...}: { + networking.firewall.allowedTCPPorts = [80 443]; + networking.firewall.allowedUDPPorts = [80 443]; +} \ No newline at end of file diff --git a/profiles/remote-builders.nix b/profiles/remote-builders.nix new file mode 100644 index 0000000..5bfbb2c --- /dev/null +++ b/profiles/remote-builders.nix @@ -0,0 +1,48 @@ +{ + lib, + config, + ... +}: let + inherit (lib.modules) mkIf; + + current_machine_hostname = config.networking.hostName; + + usb_ssh_key_file = "/usb/ssh-keys/chaos.priv"; + + ssh_key_file = + if + builtins.elem current_machine_hostname [ + "lappy-t495" + "tablet" + ] + then usb_ssh_key_file + else throw "host isn't configured for remote-builders"; + + builderDefaults = { + sshUser = "root"; + sshKey = ssh_key_file; + supportedFeatures = ["nixos-test" "benchmark" "big-parallel" "kvm"]; + mandatoryFeatures = []; + }; +in { + nix.buildMachines = [ + (mkIf (current_machine_hostname != "hetzner-vm") (builderDefaults + // { + hostName = "hetzner-vm.servers.genderfucked.monster"; + systems = ["x86_64-linux" "aarch64-linux"]; + publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU1JdDJBQnF3SGhNano5cjZhdHY0WHVYNTh4RVdlU3RrbVhVd3ZNVkd2NHcgcm9vdEBuaXhvcwo="; + maxJobs = 3; + speedFactor = 2; + })) + (mkIf (current_machine_hostname != "vault") (builderDefaults + // { + hostName = "vault.servers.genderfucked.monster"; + systems = ["x86_64-linux"]; + publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU16L1dyaG81MTFGdzhXN3FsU0NUY1V4cWh4TGlBQkJXbFNNNFRNNzJ5RWQgcm9vdEBuaXhvcwo="; + maxJobs = 2; + speedFactor = 1; + })) + ]; + nix.distributedBuilds = true; + nix.extraOptions = "builders-use-substitutes = true"; +} diff --git a/scripts/deploy-all.sh b/scripts/deploy-all.sh index 3d29394..e5cca12 100755 --- a/scripts/deploy-all.sh +++ b/scripts/deploy-all.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -set -ex +set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -REPO_ROOT="${SCRIPT_DIR}/.." -cd $REPO_ROOT +cd $SCRIPT_DIR +cd $(git rev-parse --show-toplevel) HOSTNAME=$(hostname) [ "${NO_REBUILD}" == "" ] && ./scripts/rebuild.sh $@ -[ "${HOSTNAME}" != "hetzner-vm" ] && deploy -s ".#hetzner-vm" -- $@ -[ "${HOSTNAME}" != "vault" ] && deploy -s ".#vault" -- $@ -[ "${HOSTNAME}" != "raspberry" ] && nixos-rebuild --flake .#raspberry --target-host root@192.168.0.203 $@ \ No newline at end of file +[ "${HOSTNAME}" != "hetzner-vm" ] && ./scripts/deploy/hetzner-vm.sh $@ +[ "${HOSTNAME}" != "vault" ] && ./scripts/deploy/vault.sh $@ +[ "${HOSTNAME}" != "raspberry" ] && ./scripts/deploy/raspberry.sh $@ \ No newline at end of file diff --git a/scripts/deploy/hetzner-vm.sh b/scripts/deploy/hetzner-vm.sh new file mode 100755 index 0000000..078f236 --- /dev/null +++ b/scripts/deploy/hetzner-vm.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +cd $(git rev-parse --show-toplevel) + +nixos-rebuild switch --flake .#hetzner-vm --target-host hetzner-vm -s $@ \ No newline at end of file diff --git a/scripts/deploy/raspberry.sh b/scripts/deploy/raspberry.sh new file mode 100644 index 0000000..c2512aa --- /dev/null +++ b/scripts/deploy/raspberry.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +cd $(git rev-parse --show-toplevel) + +nixos-rebuild switch --flake .#raspberry --target-host raspberry -s $@ \ No newline at end of file diff --git a/scripts/deploy/vault.sh b/scripts/deploy/vault.sh new file mode 100644 index 0000000..a76194f --- /dev/null +++ b/scripts/deploy/vault.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +cd $(git rev-parse --show-toplevel) + +nixos-rebuild switch --flake .#vault --target-host vault -s $@ \ No newline at end of file