raid & mdadm reporting for buildbox, use buildbox as remote builder

deployNodes.nix

@@ -36,4 +36,13 @@ in {
       path = activateNixOS_x64_64-linux nixosConfigurations.vault;
     };
   };
+  buildbox = {
+    hostname = "buildbox.servers.genderfucked.monster";
+    username = "root";
+    profiles.system = {
+      user = "root";
+      sshUser = "root";
+      path = activateNixOS_x64_64-linux nixosConfigurations.buildbox;
+    };
+  };
 }

flake.lock

@@ -191,11 +191,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1669052418,
-        "narHash": "sha256-M1I4BKXBQm2gey1tScemEh5TpHHE3gKptL7BpWUvL8s=",
+        "lastModified": 1669140675,
+        "narHash": "sha256-npzfyfLECsJWgzK/M4gWhykP2DNAJTYjgY2BWkz/oEQ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "20fc948445a6c22d4e8d5178e9a6bc6e1f5417c8",
+        "rev": "2788904d26dda6cfa1921c5abb7a2466ffe3cb8c",
         "type": "github"
       },
       "original": {

hosts/buildbox/buildbox.nix

@@ -1,5 +1,5 @@
 { modulesPath, tree, config, pkgs, lib, ... }:
-let secrets-db = (import ./secrets-db.nix { });
+let secrets = config.services.secrets.secrets;
 in {
   imports = with tree; [
     users.root
@@ -10,8 +10,95 @@ in {
 
     ./hardware.nix
     ./networking.nix
+    ./secrets.nix
   ];
 
+  environment.etc."mdadm.conf".text = ''
+    HOMEHOST <ignore>
+    PROGRAM /run/current-system/sw/bin/mdadm-notify
+  '';
+
+  # some taken from https://github.com/hunleyd/mdadm_notify/blob/master/mdadm_notify
+  environment.systemPackages = [
+    (pkgs.writeShellScriptBin "mdadm-notify" ''
+      event=$1
+      md_device=$2
+      device=$3
+
+      case $event in
+        DegradedArray)
+          msg="$md_device is running in DEGRADED MODE"
+          ;;
+        DeviceDisappeared)
+          msg="$md_device has DISAPPEARED"
+          ;;
+        Fail)
+          msg="$md_device had an ACTIVE component FAIL ($device)"
+          ;;
+        FailSpare)
+          msg="$md_device had a SPARE component FAIL during rebuild ($device)"
+          ;;
+        MoveSpare)
+          msg="SPARE device $device has been MOVED to a new array ($md_device)"
+          ;;
+        NewArray)
+          # silence NewArray
+          exit 0
+          msg="$md_device has APPEARED"
+          ;;
+        Rebuild??)
+          msg="$md_device REBUILD is now `echo $event|sed 's/Rebuild//'`% complete"
+          ;;
+        RebuildFinished)
+          msg="REBUILD of $md_device is COMPLETE or ABORTED"
+          ;;
+        RebuildStarted)
+          msg="RECONSTRUCTION of $md_device has STARTED"
+          ;;
+        SpareActive)
+          msg="$device has become an ACTIVE COMPONENT of $md_device"
+          ;;
+        SparesMissing)
+          msg="$md_device is MISSING one or more SPARE devices"
+          ;;
+        TestMessage)
+          msg="TEST MESSAGE generated for $md_device"
+          ;;
+      esac
+
+      printf "Subject: BuildBox mdadm: $event\n\n$msg" | msmtp "all@owo.monster"
+    '')
+  ];
+
+  programs.msmtp = {
+    enable = true;
+    accounts = {
+      default = {
+        auth = true;
+        tls = true;
+        protocol = "smtp";
+        host = "mail.owo.monster";
+        port = 587;
+        from = "system@owo.monster";
+        user = "system@owo.monster";
+        passwordeval = "cat ${secrets.system_mail_password.path}";
+      };
+    };
+  };
+
+  systemd.services.mdadm-monitor = {
+    requires = [ "network.target" ];
+    wantedBy = [ "multi-user.target" ];
+    path = with pkgs; [ mdadm msmtp ];
+    script = ''
+      exec mdadm --monitor --scan
+    '';
+    serviceConfig = {
+      Restart = "always";
+      StartLimitAction = "none";
+    };
+  };
+
   home-manager.users.root = {
     imports = with tree; [ home.base home.dev.small ];
     home.stateVersion = "22.05";

hosts/buildbox/hardware.nix

@@ -4,10 +4,7 @@
     [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
   boot.kernelModules = [ "kvm-amd" ];
 
-  environment.etc."mdadm.conf".text = ''
-    HOMEHOST <ignore>
-  '';
-  boot.initrd.services.swraid.mdadmConf  = config.environment.etc."mdadm.conf".text;
+  boot.initrd.services.swraid.mdadmConf = config.environment.etc."mdadm.conf".text;
 
   fileSystems."/" = {
     device = "/dev/disk/by-label/root";

hosts/buildbox/secrets.nix

@@ -0,0 +1,15 @@
+{ pkgs, ... }: {
+  services.secrets = {
+    enable = true;
+
+    secrets = {
+      system_mail_password = {
+        user = "root";
+        group = "root";
+        fetchScript = ''
+          simple_get "/api-keys/chaos_mail/system" .password > $secretFile
+        '';
+      };
+    };
+  };
+}

hosts/hetzner-vm/profiles/mailserver.nix

@@ -29,7 +29,12 @@ in {
         aliases = [];
         sieveScript = null;
       };
-
+      "system@owo.monster" = {
+        name = "system@owo.monster";
+        passwordFile = "${secrets.system_mail_passwd.path}";
+        aliases = [];
+        sieveScript = null;
+      };
     };
   };
 }

hosts/hetzner-vm/secrets.nix

@@ -41,6 +41,14 @@
           htpasswd -nbB "" "$password" 2>/dev/null | cut -d: -f2 > $secretFile
         '';
       };
+      system_mail_passwd = {
+        user = "dovecot2";
+        group = "dovecot2";
+        fetchScript = ''
+          password=$(simple_get "/api-keys/chaos_mail/system" .password)
+          htpasswd -nbB "" "$password" 2>/dev/null | cut -d: -f2 > $secretFile
+        '';
+      };
       gitlab_env = {
         user = "gitlab_artifacts_sync";
         group = "gitlab_artifacts_sync";

hosts/nixos.nix

@@ -68,6 +68,12 @@ in {
     modules = defaultModules ++ [ ./vault/vault.nix ];
   };
 
+  buildbox = nixosUnstableSystem {
+    specialArgs = defaultSpecialArgs;
+    system = "x86_64-linux";
+    modules = defaultModules ++ [ ./buildbox/buildbox.nix ];
+  };
+
   # nix build .#nixosConfigurations.nixos-live-x86_64.config.system.build.isoImage
   nixos-live-x86_64 = nixosUnstableSystem {
     specialArgs = defaultSpecialArgs;

hosts/tablet/tablet.nix

@@ -35,7 +35,24 @@
   networking.firewall.enable = true;
   networking.firewall.allowPing = true;
 
-  networking.enableIPv6 = false;
+  networking.enableIPv6 = true;
+
+  nix.buildMachines = [{
+    hostName = "buildbox.servers.genderfucked.monster";
+    system = "x86_64-linux";
+    # if the builder supports building for multiple architectures, 
+    # replace the previous line by, e.g.,
+    # systems = ["x86_64-linux" "aarch64-linux"];
+    sshUser = "root";
+    sshKey = "/usb/ssh-keys/chaos.priv";
+    publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpXZGI5SVl3dFBSRm9rK2JTWUpmSnlRTlJSSithVEtIT3VOTkNLY2FMUHggcm9vdEBuaXhvcwo=";
+    maxJobs = 16;
+    speedFactor = 4;
+    supportedFeatures = [ "nixos-test" "benchmark" "big-parallel" "kvm" ];
+    mandatoryFeatures = [ ];
+  }];
+  nix.distributedBuilds = true;
+  nix.extraOptions = "builders-use-substitutes = true";
 
   networking.hostName = "tablet";
   time.timeZone = "Europe/London";

scripts/deploy-all.sh

@@ -8,7 +8,9 @@ cd $REPO_ROOT
 
 HOSTNAME=$(hostname)
 
-./scripts/rebuild.sh $@
+[ "${NO_REBUILD}" == "" ] && ./scripts/rebuild.sh $@
 [ "${HOSTNAME}" != "hetzner-vm" ] && deploy -s ".#hetzner-vm" -- $@
-[ "${HOSTNAME}" != "vault" ] deploy -s ".#vault" -- $@
+[ "${HOSTNAME}" != "vault" ] && deploy -s ".#vault" -- $@
 [ "${HOSTNAME}" != "storage" ] && deploy -s ".#storage" -- $@
+[ "${HOSTNAME}" != "buildbox" ] && deploy -s ".#buildbox" -- $@
+