chaos/nixfiles
1
0
Fork 0
Code Issues 2 Releases Activity

move storage container to profiles

Browse source
This commit is contained in:
chaos 2025-02-02 16:36:01 +00:00
parent dd059f31b0
commit 5978e8d9de
No known key found for this signature in database
11 changed files with 189 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
hosts/hetzner-arm/containers/storage/data/ports.nix
View file

@ -1,16 +0,0 @@
{
webdav = let
base = 4200;
in {
public = base + 0;
uploads = base + 1;
};
http = let
base = 4300;
in {
music = base + 0;
public = base + 1;
uploads_public = base + 2;
};
}

70
hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix
View file

@ -1,70 +0,0 @@
{config, ...}: let
inherit (config.services.secrets) secrets;
ports = import ../data/ports.nix;
in {
systemd.tmpfiles.rules = [
"d /caches - storage storage"
"d /caches/main_webdav_serve - storage storage"
"d /caches/media_webdav_serve - storage storage"
];
services.rclone-serve = {
enable = true;
remotes = map (remote:
{
user = "storage";
}
// remote) [
{
id = "public";
remote = "Public:";
type = "webdav";
extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav.public}"
"--htpasswd=${secrets.webdav_public_htpasswd.path}"
"--baseurl=/Public/"
];
}
{
id = "uploads";
remote = "Uploads:";
type = "webdav";
extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav.uploads}"
"--htpasswd=${secrets.webdav_uploads_htpasswd.path}"
"--baseurl=/Uploads/"
];
}
{
id = "music";
remote = "Music:";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.music}"
"--baseurl=/Music/"
"--read-only"
];
}
{
id = "public";
remote = "Public:";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.public}"
"--baseurl=/Public/"
"--read-only"
];
}
{
id = "uploads_public";
remote = "Uploads:Public";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.uploads_public}"
"--baseurl=/Uploads/"
"--read-only"
];
}
];
};
}

10
hosts/hetzner-arm/containers/storage/profiles/users.nix
View file

@ -1,10 +0,0 @@
{...}: {
users.groups.storage = {
gid = 1000;
};
users.users.storage = {
uid = 1000;
isNormalUser = true;
extraGroups = ["storage"];
};
}

63
hosts/hetzner-arm/containers/storage/secrets.nix
View file

@ -1,63 +0,0 @@
{pkgs, ...}: {
services.secrets = {
enable = true;
vaultLogin = {
enable = true;
loginUsername = "hetzner-arm-container-storage";
};
requiredVaultPaths = [
"api-keys/data/storage/webdav/Public"
"api-keys/data/storage/webdav/Uploads"
];
packages = with pkgs; [
apacheHttpd
];
extraFunctions = ''
simple_get_htpasswd() {
if [ -f "$2" ]; then
rm "$2"
fi
touch "$2"
data=$(kv_get "$1" | base64)
for username in $(echo "$data" | base64 -d | jq -r ".data.data | keys | .[]"); do
password=$(echo "$data" | base64 -d | jq -r ".data.data.\"$username\"")
htpasswd -b "$2" "$username" "$password" 2>/dev/null
done
}
'';
secrets = {
vault_password = {
manual = true;
};
webdav_public_htpasswd = {
user = "storage";
group = "storage";
fetchScript = ''
simple_get_htpasswd "/api-keys/storage/webdav/Public" "$secretFile"
'';
};
webdav_uploads_htpasswd = {
user = "storage";
group = "storage";
fetchScript = ''
simple_get_htpasswd "/api-keys/storage/webdav/Uploads" "$secretFile"
'';
};
rclone_config = {
user = "storage";
group = "storage";
manual = true;
};
};
};
}

110
hosts/hetzner-arm/containers/storage/storage.nix
View file

@ -1,110 +0,0 @@
{
self,
tree,
lib,
inputs,
pkgs,
...
}: let
inherit (lib.attrsets) attrValues;
inherit (lib.lists) flatten;
containerAddresses = import "${self}/hosts/hetzner-arm/data/containerAddresses.nix";
hostIP = containerAddresses.host;
containerIP = containerAddresses.containers.storage;
clientMaxBodySize = "${toString (1024 * 128)}M";
ports = import ./data/ports.nix;
in {
containers.storage = {
autoStart = true;
privateNetwork = true;
hostAddress = hostIP;
localAddress = containerIP;
bindMounts = {
"/dev/fuse" = {
hostPath = "/dev/fuse";
isReadOnly = false;
};
};
# Allow rclone mount in container
allowedDevices = [
{
modifier = "rwm";
node = "/dev/fuse";
}
{
modifier = "rwm";
node = "/dev/mapper/control";
}
];
specialArgs = {
inherit inputs;
inherit tree;
inherit self;
};
config = {...}: {
nixpkgs.pkgs = pkgs;
imports = flatten (with tree; [
presets.nixos.containerBase
(with hosts.hetzner-arm.containers.storage.profiles; [
rcloneConfigs
rcloneServe
users
])
./secrets.nix
]);
environment.systemPackages = with pkgs; [
rclone
fuse
fuse3
];
networking.firewall = {
enable = true;
allowedTCPPorts = flatten [
(attrValues ports.webdav)
(attrValues ports.http)
];
};
home-manager.users.root.home.stateVersion = "25.05";
system.stateVersion = "25.05";
};
};
services.nginx.virtualHosts."storage-webdav.owo.monster" = {
forceSSL = true;
enableACME = true;
locations = {
"/Public/".proxyPass = "http://${containerIP}:${toString ports.webdav.public}";
"/Uploads/".proxyPass = "http://${containerIP}:${toString ports.webdav.uploads}";
};
extraConfig = ''
client_max_body_size ${clientMaxBodySize};
'';
};
services.nginx.virtualHosts."storage-http.owo.monster" = {
forceSSL = true;
enableACME = true;
locations = {
"/Music/".proxyPass = "http://${containerIP}:${toString ports.http.music}";
"/Public/".proxyPass = "http://${containerIP}:${toString ports.http.public}";
"/Uploads/".proxyPass = "http://${containerIP}:${toString ports.http.uploads_public}";
};
extraConfig = ''
client_max_body_size ${clientMaxBodySize};
'';
};
}

25
hosts/hetzner-arm/hetzner-arm.nix
View file

@ -3,7 +3,7 @@
lib,
...
}: let
inherit (lib.lists) forEach flatten;
inherit (lib.lists) flatten;
in {
imports = flatten (with tree; [
(with tree.presets.nixos; [
@ -15,20 +15,18 @@ in {
profiles.nixos.nginx
(forEach [
"storage"
] (name: ./containers + "/${name}/${name}.nix"))
(with hosts.hetzner-arm.profiles; [
staticSites
gotosocial
forgejo
mpd
radicale
vault
restic
vaultwarden
photoprism
radicale
rclone
restic
staticSites
storage
vault
vaultwarden
])
./hardware.nix
@ -44,13 +42,6 @@ in {
})
];
# For Containers
networking.nat = {
enable = true;
internalInterfaces = ["ve-+"];
externalInterface = "enp1s0";
};
networking.firewall = {
allowedTCPPorts = [80 443];
allowedUDPPorts = [80 443];

1
hosts/hetzner-arm/profiles/mpd.nix
View file

@ -20,6 +20,7 @@
in {
environment.systemPackages = with pkgs; [
mpc_cli
musicutil
];
systemd.tmpfiles.rules = [

3
hosts/hetzner-arm/profiles/photoprism.nix
View file

@ -7,7 +7,7 @@
inherit (config.services.secrets) secrets;
rclonePhotos = pkgs.writeShellScriptBin "rclone-photos" ''
${pkgs.rclone}/bin/rclone --config ${secrets.photos_rclone_config.path} "$@"
${pkgs.rclone}/bin/rclone --config ${secrets.rclone_config.path} "$@"
'';
mountPhotos = pkgs.writeShellScriptBin "mount-photos" ''
@ -67,6 +67,7 @@ in {
isSystemUser = true;
uid = 1290;
group = "photoprism";
extraGroups = ["storage"];
};
users.groups.photoprism.gid = 1290;

27
hosts/hetzner-arm/containers/storage/profiles/rcloneConfigs.nix → hosts/hetzner-arm/profiles/rclone.nix
View file

@ -1,6 +1,31 @@
{config, ...}: let
{
config,
pkgs,
...
}: let
inherit (config.services.secrets) secrets;
rcloneStorage = pkgs.writeShellScriptBin "rclone-storage" ''
${pkgs.rclone}/bin/rclone --config ${secrets.rclone_config.path} "$@"
'';
in {
environment.systemPackages = with pkgs; [
rclone
fuse
fuse3
rcloneStorage
];
users.groups.storage = {
gid = 1000;
};
users.users.storage = {
uid = 1000;
isNormalUser = true;
extraGroups = ["storage"];
};
systemd.tmpfiles.rules = [
"d /root/.config - root root"
"d /root/.config/rclone - root root"

106
hosts/hetzner-arm/profiles/storage.nix
View file

@ -0,0 +1,106 @@
{config, ...}: let
inherit (config.services.secrets) secrets;
ports = {
webdav = let
base = 4200;
in {
public = base + 0;
uploads = base + 1;
};
http = let
base = 4300;
in {
music = base + 0;
public = base + 1;
uploads_public = base + 2;
};
};
clientMaxBodySize = "${toString (1024 * 128)}M";
in {
services.rclone-serve = {
enable = true;
remotes = map (remote:
{
user = "storage";
}
// remote) [
{
id = "public";
remote = "Public:";
type = "webdav";
extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav.public}"
"--htpasswd=${secrets.webdav_public_htpasswd.path}"
"--baseurl=/Public/"
];
}
{
id = "uploads";
remote = "Uploads:";
type = "webdav";
extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav.uploads}"
"--htpasswd=${secrets.webdav_uploads_htpasswd.path}"
"--baseurl=/Uploads/"
];
}
{
id = "music";
remote = "Music:";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.music}"
"--baseurl=/Music/"
"--read-only"
];
}
{
id = "public";
remote = "Public:";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.public}"
"--baseurl=/Public/"
"--read-only"
];
}
{
id = "uploads_public";
remote = "Uploads:Public";
type = "http";
extraArgs = [
"--addr=0.0.0.0:${toString ports.http.uploads_public}"
"--baseurl=/Uploads/"
"--read-only"
];
}
];
};
services.nginx.virtualHosts."storage-webdav.owo.monster" = {
forceSSL = true;
enableACME = true;
locations = {
"/Public/".proxyPass = "http://127.0.0.1:${toString ports.webdav.public}";
"/Uploads/".proxyPass = "http://127.0.0.1:${toString ports.webdav.uploads}";
};
extraConfig = ''
client_max_body_size ${clientMaxBodySize};
'';
};
services.nginx.virtualHosts."storage-http.owo.monster" = {
forceSSL = true;
enableACME = true;
locations = {
"/Music/".proxyPass = "http://127.0.0.1:${toString ports.http.music}";
"/Public/".proxyPass = "http://127.0.0.1:${toString ports.http.public}";
"/Uploads/".proxyPass = "http://127.0.0.1:${toString ports.http.uploads_public}";
};
extraConfig = ''
client_max_body_size ${clientMaxBodySize};
'';
};
}

128
hosts/hetzner-arm/secrets.nix
View file

@ -9,62 +9,43 @@
packages = with pkgs; [
apacheHttpd
rclone
];
extraFunctions = ''
simple_get_htpasswd() {
if [ -f "$2" ]; then
rm "$2"
fi
touch "$2"
data=$(kv_get "$1" | base64)
for username in $(echo "$data" | base64 -d | jq -r ".data.data | keys | .[]"); do
password=$(echo "$data" | base64 -d | jq -r ".data.data.\"$username\"")
htpasswd -b "$2" "$username" "$password" 2>/dev/null
done
}
'';
requiredVaultPaths = [
"private-public-keys/data/ssh/root@hetzner-arm"
"private-public-keys/data/ssh/root@hetzner-arm-decrypt"
"api-keys/data/backblaze/Backblaze"
"private-public-keys/data/restic/Social-02"
"private-public-keys/data/restic/Forgejo"
"private-public-keys/data/rclone/Chaos-Photos-Crypt"
"private-public-keys/data/restic/Radicale"
"private-public-keys/data/restic/Vault"
"api-keys/data/mpd"
"api-keys/data/music-stream"
"api-keys/data/radicale"
"private-public-keys/data/restic/Radicale"
"private-public-keys/data/restic/Vault"
"api-keys/data/storage/webdav/Public"
"api-keys/data/storage/webdav/Uploads"
];
extraFunctions = ''
replace_slash_for_sed() {
sed "s#/#\\\/#"
}
simple_get_obscure() {
rclone obscure "$(simple_get "$@")"
}
simple_get_replace_b2() {
api_account=$(simple_get "$1" .keyID | replace_slash_for_sed)
api_key=$(simple_get "$1" .applicationKey | replace_slash_for_sed)
replace_account=''${2}_ACCOUNT
replace_key=''${2}_KEY
sed -i "s/$replace_account/$api_account/" "$3"
sed -i "s/$replace_key/$api_key/" "$3"
}
simple_get_replace_crypt() {
password=$(simple_get_obscure "$1" .password)
salt=$(simple_get_obscure "$1" .salt)
replace_password=''${2}_PASSWORD
replace_salt=''${2}_SALT
sed -i "s/$replace_password/$password/" "$3"
sed -i "s/$replace_salt/$salt/" "$3"
}
'';
secrets = {
vault_password = {
manual = true;
@ -120,6 +101,18 @@
'';
};
restic_password_radicale = {
fetchScript = ''
simple_get "/private-public-keys/restic/Radicale" .password > "$secretFile"
'';
};
restic_password_vault = {
fetchScript = ''
simple_get "/private-public-keys/restic/Vault" .password > "$secretFile"
'';
};
mpd_control_password = {
user = "mpd";
group = "mpd";
@ -141,60 +134,31 @@
radicale_htpasswd = {
user = "radicale";
group = "radicale";
fetchScript = ''
if [ -f "$secretFile" ]; then
rm "$secretFile"
fi
touch "$secretFile"
data=$(kv_get "/api-keys/radicale" | base64)
for username in $(echo "$data" | base64 -d | jq -r ".data.data | keys | .[]"); do
password=$(echo "$data" | base64 -d | jq -r ".data.data.\"$username\"")
htpasswd -bB "$secretFile" "$username" "$password" 2>/dev/null
done
fetchScript = '' $secretFile"
simple_get_htpasswd "/api-keys/radicale" $secretFile"
'';
};
restic_password_radicale = {
webdav_public_htpasswd = {
user = "storage";
group = "storage";
fetchScript = ''
simple_get "/private-public-keys/restic/Radicale" .password > "$secretFile"
simple_get_htpasswd "/api-keys/storage/webdav/Public" "$secretFile"
'';
};
restic_password_vault = {
webdav_uploads_htpasswd = {
user = "storage";
group = "storage";
fetchScript = ''
simple_get "/private-public-keys/restic/Vault" .password > "$secretFile"
simple_get_htpasswd "/api-keys/storage/webdav/Uploads" "$secretFile"
'';
};
photos_rclone_config = {
user = "photoprism";
group = "photoprism";
fetchScript = let
template = builtins.toFile "template.conf" ''
[B2]
type = b2
account = B2_ACCOUNT
key = B2_KEY
hard_delete = true
[Photos-Crypt]
type = crypt
remote = B2:Chaos-Photos
password = PHOTOS_CRYPT_PASSWORD
password2 = PHOTOS_CRYPT_SALT
[Photos]
type = chunker
remote = Photos-Crypt:
chunk_size = 128Mi
'';
in ''
cat ${template} > "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Backblaze" "B2" "$secretFile"
simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Photos-Crypt" "PHOTOS_CRYPT" "$secretFile"
'';
rclone_config = {
user = "storage";
group = "storage";
manual = true;
};
};
};