From 4d356ec79ec19df1cafc2742d32bcfae642ceeb4 Mon Sep 17 00:00:00 2001 From: Chaos Date: Thu, 24 Nov 2022 13:29:48 +0000 Subject: [PATCH] raid & mdadm reporting for buildbox, use buildbox as remote builder --- deployNodes.nix | 9 +++ flake.lock | 6 +- hosts/buildbox/buildbox.nix | 89 +++++++++++++++++++++++- hosts/buildbox/hardware.nix | 5 +- hosts/buildbox/secrets.nix | 15 ++++ hosts/hetzner-vm/profiles/mailserver.nix | 7 +- hosts/hetzner-vm/secrets.nix | 8 +++ hosts/nixos.nix | 6 ++ hosts/tablet/tablet.nix | 19 ++++- scripts/deploy-all.sh | 6 +- 10 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 hosts/buildbox/secrets.nix diff --git a/deployNodes.nix b/deployNodes.nix index 0a666b4..84d9674 100644 --- a/deployNodes.nix +++ b/deployNodes.nix @@ -36,4 +36,13 @@ in { path = activateNixOS_x64_64-linux nixosConfigurations.vault; }; }; + buildbox = { + hostname = "buildbox.servers.genderfucked.monster"; + username = "root"; + profiles.system = { + user = "root"; + sshUser = "root"; + path = activateNixOS_x64_64-linux nixosConfigurations.buildbox; + }; + }; } diff --git a/flake.lock b/flake.lock index eb675ef..f20a670 100644 --- a/flake.lock +++ b/flake.lock @@ -191,11 +191,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1669052418, - "narHash": "sha256-M1I4BKXBQm2gey1tScemEh5TpHHE3gKptL7BpWUvL8s=", + "lastModified": 1669140675, + "narHash": "sha256-npzfyfLECsJWgzK/M4gWhykP2DNAJTYjgY2BWkz/oEQ=", "owner": "nixos", "repo": "nixpkgs", - "rev": "20fc948445a6c22d4e8d5178e9a6bc6e1f5417c8", + "rev": "2788904d26dda6cfa1921c5abb7a2466ffe3cb8c", "type": "github" }, "original": { diff --git a/hosts/buildbox/buildbox.nix b/hosts/buildbox/buildbox.nix index cb74434..61172a2 100644 --- a/hosts/buildbox/buildbox.nix +++ b/hosts/buildbox/buildbox.nix @@ -1,5 +1,5 @@ { modulesPath, tree, config, pkgs, lib, ... }: -let secrets-db = (import ./secrets-db.nix { }); +let secrets = config.services.secrets.secrets; in { imports = with tree; [ users.root @@ -10,8 +10,95 @@ in { ./hardware.nix ./networking.nix + ./secrets.nix ]; + environment.etc."mdadm.conf".text = '' + HOMEHOST + PROGRAM /run/current-system/sw/bin/mdadm-notify + ''; + + # some taken from https://github.com/hunleyd/mdadm_notify/blob/master/mdadm_notify + environment.systemPackages = [ + (pkgs.writeShellScriptBin "mdadm-notify" '' + event=$1 + md_device=$2 + device=$3 + + case $event in + DegradedArray) + msg="$md_device is running in DEGRADED MODE" + ;; + DeviceDisappeared) + msg="$md_device has DISAPPEARED" + ;; + Fail) + msg="$md_device had an ACTIVE component FAIL ($device)" + ;; + FailSpare) + msg="$md_device had a SPARE component FAIL during rebuild ($device)" + ;; + MoveSpare) + msg="SPARE device $device has been MOVED to a new array ($md_device)" + ;; + NewArray) + # silence NewArray + exit 0 + msg="$md_device has APPEARED" + ;; + Rebuild??) + msg="$md_device REBUILD is now `echo $event|sed 's/Rebuild//'`% complete" + ;; + RebuildFinished) + msg="REBUILD of $md_device is COMPLETE or ABORTED" + ;; + RebuildStarted) + msg="RECONSTRUCTION of $md_device has STARTED" + ;; + SpareActive) + msg="$device has become an ACTIVE COMPONENT of $md_device" + ;; + SparesMissing) + msg="$md_device is MISSING one or more SPARE devices" + ;; + TestMessage) + msg="TEST MESSAGE generated for $md_device" + ;; + esac + + printf "Subject: BuildBox mdadm: $event\n\n$msg" | msmtp "all@owo.monster" + '') + ]; + + programs.msmtp = { + enable = true; + accounts = { + default = { + auth = true; + tls = true; + protocol = "smtp"; + host = "mail.owo.monster"; + port = 587; + from = "system@owo.monster"; + user = "system@owo.monster"; + passwordeval = "cat ${secrets.system_mail_password.path}"; + }; + }; + }; + + systemd.services.mdadm-monitor = { + requires = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ mdadm msmtp ]; + script = '' + exec mdadm --monitor --scan + ''; + serviceConfig = { + Restart = "always"; + StartLimitAction = "none"; + }; + }; + home-manager.users.root = { imports = with tree; [ home.base home.dev.small ]; home.stateVersion = "22.05"; diff --git a/hosts/buildbox/hardware.nix b/hosts/buildbox/hardware.nix index 12fb568..66a8cc2 100644 --- a/hosts/buildbox/hardware.nix +++ b/hosts/buildbox/hardware.nix @@ -4,10 +4,7 @@ [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ]; boot.kernelModules = [ "kvm-amd" ]; - environment.etc."mdadm.conf".text = '' - HOMEHOST - ''; - boot.initrd.services.swraid.mdadmConf = config.environment.etc."mdadm.conf".text; + boot.initrd.services.swraid.mdadmConf = config.environment.etc."mdadm.conf".text; fileSystems."/" = { device = "/dev/disk/by-label/root"; diff --git a/hosts/buildbox/secrets.nix b/hosts/buildbox/secrets.nix new file mode 100644 index 0000000..8711487 --- /dev/null +++ b/hosts/buildbox/secrets.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: { + services.secrets = { + enable = true; + + secrets = { + system_mail_password = { + user = "root"; + group = "root"; + fetchScript = '' + simple_get "/api-keys/chaos_mail/system" .password > $secretFile + ''; + }; + }; + }; +} diff --git a/hosts/hetzner-vm/profiles/mailserver.nix b/hosts/hetzner-vm/profiles/mailserver.nix index c7e0dfd..11b4223 100644 --- a/hosts/hetzner-vm/profiles/mailserver.nix +++ b/hosts/hetzner-vm/profiles/mailserver.nix @@ -29,7 +29,12 @@ in { aliases = []; sieveScript = null; }; - + "system@owo.monster" = { + name = "system@owo.monster"; + passwordFile = "${secrets.system_mail_passwd.path}"; + aliases = []; + sieveScript = null; + }; }; }; } diff --git a/hosts/hetzner-vm/secrets.nix b/hosts/hetzner-vm/secrets.nix index cc2e9d8..29f4309 100644 --- a/hosts/hetzner-vm/secrets.nix +++ b/hosts/hetzner-vm/secrets.nix @@ -41,6 +41,14 @@ htpasswd -nbB "" "$password" 2>/dev/null | cut -d: -f2 > $secretFile ''; }; + system_mail_passwd = { + user = "dovecot2"; + group = "dovecot2"; + fetchScript = '' + password=$(simple_get "/api-keys/chaos_mail/system" .password) + htpasswd -nbB "" "$password" 2>/dev/null | cut -d: -f2 > $secretFile + ''; + }; gitlab_env = { user = "gitlab_artifacts_sync"; group = "gitlab_artifacts_sync"; diff --git a/hosts/nixos.nix b/hosts/nixos.nix index 2be64e7..449c251 100644 --- a/hosts/nixos.nix +++ b/hosts/nixos.nix @@ -68,6 +68,12 @@ in { modules = defaultModules ++ [ ./vault/vault.nix ]; }; + buildbox = nixosUnstableSystem { + specialArgs = defaultSpecialArgs; + system = "x86_64-linux"; + modules = defaultModules ++ [ ./buildbox/buildbox.nix ]; + }; + # nix build .#nixosConfigurations.nixos-live-x86_64.config.system.build.isoImage nixos-live-x86_64 = nixosUnstableSystem { specialArgs = defaultSpecialArgs; diff --git a/hosts/tablet/tablet.nix b/hosts/tablet/tablet.nix index 0f07a32..c5b2c0e 100644 --- a/hosts/tablet/tablet.nix +++ b/hosts/tablet/tablet.nix @@ -35,7 +35,24 @@ networking.firewall.enable = true; networking.firewall.allowPing = true; - networking.enableIPv6 = false; + networking.enableIPv6 = true; + + nix.buildMachines = [{ + hostName = "buildbox.servers.genderfucked.monster"; + system = "x86_64-linux"; + # if the builder supports building for multiple architectures, + # replace the previous line by, e.g., + # systems = ["x86_64-linux" "aarch64-linux"]; + sshUser = "root"; + sshKey = "/usb/ssh-keys/chaos.priv"; + publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpXZGI5SVl3dFBSRm9rK2JTWUpmSnlRTlJSSithVEtIT3VOTkNLY2FMUHggcm9vdEBuaXhvcwo="; + maxJobs = 16; + speedFactor = 4; + supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ]; + mandatoryFeatures = [ ]; + }]; + nix.distributedBuilds = true; + nix.extraOptions = "builders-use-substitutes = true"; networking.hostName = "tablet"; time.timeZone = "Europe/London"; diff --git a/scripts/deploy-all.sh b/scripts/deploy-all.sh index e0d9195..99354b5 100755 --- a/scripts/deploy-all.sh +++ b/scripts/deploy-all.sh @@ -8,7 +8,9 @@ cd $REPO_ROOT HOSTNAME=$(hostname) -./scripts/rebuild.sh $@ +[ "${NO_REBUILD}" == "" ] && ./scripts/rebuild.sh $@ [ "${HOSTNAME}" != "hetzner-vm" ] && deploy -s ".#hetzner-vm" -- $@ -[ "${HOSTNAME}" != "vault" ] deploy -s ".#vault" -- $@ +[ "${HOSTNAME}" != "vault" ] && deploy -s ".#vault" -- $@ [ "${HOSTNAME}" != "storage" ] && deploy -s ".#storage" -- $@ +[ "${HOSTNAME}" != "buildbox" ] && deploy -s ".#buildbox" -- $@ +