diff --git a/hosts/hetzner-arm/containers/jellyfin/data/rclone_config.template b/hosts/hetzner-arm/containers/jellyfin/data/rclone_config.template index e53956b..ba06284 100644 --- a/hosts/hetzner-arm/containers/jellyfin/data/rclone_config.template +++ b/hosts/hetzner-arm/containers/jellyfin/data/rclone_config.template @@ -1,6 +1,31 @@ -[Media] +[PutIO-WebDAV] type = webdav -url = https://storage-webdav.owo.monster/Media/ +url = https://webdav.put.io vendor = other -user = media -pass = MEDIA_PASSWORD \ No newline at end of file +user = chaoticryptidz +pass = PUTIO_PASSWORD + +[B2-Media-Source] +type = b2 +account = B2_CHAOS_MEDIA_ACCOUNT +key = B2_CHAOS_MEDIA_KEY +hard_delete = true + +[B2-Media] +type = alias +remote = B2-Media-Source:Chaos-Media + +[Media-Source] +type = crypt +remote = B2-Media: +password = STORAGE_MEDIA_CRYPT_PASSWORD +password2 = STORAGE_MEDIA_CRYPT_SALT + +[Media] +type = chunker +remote = Media-Source: +chunk_size = 256Mi + +[Media-Combine] +type = combine +upstreams = "Media=Media:" "PutIO=PutIO-WebDAV:" \ No newline at end of file diff --git a/hosts/hetzner-arm/containers/jellyfin/profiles/mediaMount.nix b/hosts/hetzner-arm/containers/jellyfin/profiles/mediaMount.nix index 32b7005..d99458f 100644 --- a/hosts/hetzner-arm/containers/jellyfin/profiles/mediaMount.nix +++ b/hosts/hetzner-arm/containers/jellyfin/profiles/mediaMount.nix @@ -8,12 +8,16 @@ ${pkgs.rclone}/bin/rclone --config ${secrets.rclone_config.path} "$@" ''; mountMedia = pkgs.writeShellScriptBin "mount-media" '' - ${rcloneMedia}/bin/rclone-media mount Media: /Media \ + ${rcloneMedia}/bin/rclone-media mount Media-Combine: /Media \ --allow-other \ --uid=${toString config.users.users.jellyfin.uid} \ --gid=${toString config.users.groups.jellyfin.gid} \ --fast-list \ --umask=666 \ + --dir-cache-time=1m \ + --vfs-cache-mode=full \ + --vfs-cache-max-age=120m \ + --vfs-cache-max-size=5g \ --log-level=INFO "$@" ''; in { diff --git a/hosts/hetzner-arm/containers/jellyfin/secrets.nix b/hosts/hetzner-arm/containers/jellyfin/secrets.nix index 7654849..6aac993 100644 --- a/hosts/hetzner-arm/containers/jellyfin/secrets.nix +++ b/hosts/hetzner-arm/containers/jellyfin/secrets.nix @@ -3,9 +3,42 @@ enable = true; packages = with pkgs; [ + # for rclone obscure config file rclone ]; + extraFunctions = '' + replace_slash_for_sed() { + sed "s#/#\\\/#" + } + + simple_get_obscure() { + rclone obscure "$(simple_get "$@")" + } + + simple_get_replace_b2() { + api_account=$(simple_get "$1" .keyID | replace_slash_for_sed) + api_key=$(simple_get "$1" .applicationKey | replace_slash_for_sed) + + replace_account=''${2}_ACCOUNT + replace_key=''${2}_KEY + + sed -i "s/$replace_account/$api_account/" "$3" + sed -i "s/$replace_key/$api_key/" "$3" + } + + simple_get_replace_crypt() { + password=$(simple_get_obscure "$1" .password) + salt=$(simple_get_obscure "$1" .salt) + + replace_password=''${2}_PASSWORD + replace_salt=''${2}_SALT + + sed -i "s/$replace_password/$password/" "$3" + sed -i "s/$replace_salt/$salt/" "$3" + } + ''; + vaultLogin = { enable = true; loginUsername = "hetzner-arm-container-jellyfin"; @@ -16,9 +49,11 @@ }; requiredVaultPaths = [ - "api-keys/data/storage/webdav/Media" - "api-keys/data/storage/restic/Jellyfin" + "api-keys/data/backblaze/Chaos-Media" + "api-keys/data/putio" + "private-public-keys/data/rclone/Chaos-Media-Crypt" + "api-keys/data/storage/restic/Jellyfin" "private-public-keys/data/restic/Jellyfin" ]; @@ -32,9 +67,14 @@ group = "jellyfin"; fetchScript = '' cp ${./data/rclone_config.template} "$secretFile" - MEDIA_PASSWORD="$(simple_get "/api-keys/storage/webdav/Media" .media)" - MEDIA_PASSWORD="$(rclone obscure "$MEDIA_PASSWORD")" - sed -i "s/MEDIA_PASSWORD/$MEDIA_PASSWORD/" "$secretFile" + + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Media" "B2_CHAOS_MEDIA" "$secretFile" + + PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)" + PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")" + sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" "$secretFile" + + simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" "$secretFile" ''; }; diff --git a/hosts/hetzner-arm/containers/social/profiles/gotosocial.nix b/hosts/hetzner-arm/containers/social/profiles/gotosocial.nix index 042a8ca..96ec0fb 100644 --- a/hosts/hetzner-arm/containers/social/profiles/gotosocial.nix +++ b/hosts/hetzner-arm/containers/social/profiles/gotosocial.nix @@ -41,11 +41,11 @@ in { smtp-host = "mail.owo.monster"; smtp-port = 587; + smtp-from = "gotosocial@owo.monster"; smtp-username = "gotosocial@owo.monster"; smtp-password = ""; # set via env variables - smtp-from = "gotosocial@owo.monster"; - media-remote-cache-days = 2; + media-remote-cache-days = 1; media-image-max-size = 1000000 * 64; # MB media-video-max-size = 1000000 * 1024; # MB @@ -64,7 +64,7 @@ in { # instance-inject-mastodon-version = true; cache = { - memory-target = "256MiB"; + memory-target = "512MiB"; }; }; }; diff --git a/hosts/hetzner-arm/containers/social/profiles/restic.nix b/hosts/hetzner-arm/containers/social/profiles/restic.nix index 55821f0..d5b3aa4 100644 --- a/hosts/hetzner-arm/containers/social/profiles/restic.nix +++ b/hosts/hetzner-arm/containers/social/profiles/restic.nix @@ -23,7 +23,7 @@ ${goToSocialAdmin} export --path /var/lib/gotosocial/gts-export.json - ${goToSocialAdmin} media prune all + ${goToSocialAdmin} media prune all --dry-run=false systemctl start gotosocial '') diff --git a/hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix b/hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix index 35b7da7..a5c184b 100644 --- a/hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix +++ b/hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix @@ -33,8 +33,10 @@ in { ]; } { + # This isn't really being directly used anymore but its nice as a backup + # Jellyfin directly mounts from StorageBox instead id = "media-combine"; - remote = "Media-Combine-Serve:"; + remote = "Media-Combine:"; type = "webdav"; extraArgs = [ "--addr=0.0.0.0:${toString ports.webdav_media}" diff --git a/hosts/hetzner-arm/containers/storage/rclone_config.template b/hosts/hetzner-arm/containers/storage/rclone_config.template index 0930118..6e20faa 100644 --- a/hosts/hetzner-arm/containers/storage/rclone_config.template +++ b/hosts/hetzner-arm/containers/storage/rclone_config.template @@ -99,12 +99,27 @@ vendor = other user = chaoticryptidz pass = PUTIO_PASSWORD -[Storage-Media-Crypt] +[B2-Media-Source] +type = b2 +account = B2_CHAOS_MEDIA_ACCOUNT +key = B2_CHAOS_MEDIA_KEY +hard_delete = true + +[B2-Media] +type = alias +remote = B2-Media-Source:Chaos-Media + +[Media-Source] type = crypt -remote = StorageBox:Media +remote = B2-Media: password = STORAGE_MEDIA_CRYPT_PASSWORD password2 = STORAGE_MEDIA_CRYPT_SALT -[Media-Combine-Serve] +[Media] +type = chunker +remote = Media-Source: +chunk_size = 256Mi + +[Media-Combine] type = combine -upstreams = "Media=Storage-Media-Crypt:" "PutIO=PutIO-WebDAV:" +upstreams = "Media=Media:" "PutIO=PutIO-WebDAV:" diff --git a/hosts/hetzner-arm/containers/storage/secrets.nix b/hosts/hetzner-arm/containers/storage/secrets.nix index 599f71c..e6a43df 100644 --- a/hosts/hetzner-arm/containers/storage/secrets.nix +++ b/hosts/hetzner-arm/containers/storage/secrets.nix @@ -21,6 +21,7 @@ "api-keys/data/backblaze/Chaos-Music" "api-keys/data/backblaze/Chaos-Personal" "api-keys/data/backblaze/Chaos-Public" + "api-keys/data/backblaze/Chaos-Media" "api-keys/data/backblaze/Phoenix-Cryptidz-Storage" "api-keys/data/storage/restic/Music" @@ -211,33 +212,24 @@ user = "storage"; group = "storage"; fetchScript = '' - TMP_DIR="$(mktemp -d)" - - cp ${./rclone_config.template} "$TMP_DIR/template" - - pushd "$TMP_DIR" >/dev/null + cp ${./rclone_config.template} "$secretFile" STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password) - sed -i "s/STORAGEBOX_PASSWORD/$STORAGEBOX_PASSWORD/" ./template + sed -i "s/STORAGEBOX_PASSWORD/$STORAGEBOX_PASSWORD/" "$secretFile" - simple_get_replace_b2 "/api-keys/backblaze/Chaos-Backups" "B2_CHAOS_BACKUPS" ./template - simple_get_replace_b2 "/api-keys/backblaze/Chaos-Photos" "B2_CHAOS_PHOTOS" ./template - simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" ./template - simple_get_replace_b2 "/api-keys/backblaze/Chaos-Personal" "B2_CHAOS_PERSONAL" ./template - simple_get_replace_b2 "/api-keys/backblaze/Chaos-Public" "B2_CHAOS_PUBLIC" ./template - simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" ./template + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Media" "B2_CHAOS_MEDIA" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Backups" "B2_CHAOS_BACKUPS" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Photos" "B2_CHAOS_PHOTOS" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Personal" "B2_CHAOS_PERSONAL" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Chaos-Public" "B2_CHAOS_PUBLIC" "$secretFile" + simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" "$secretFile" PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)" PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")" - sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" ./template + sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" "$secretFile" - simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" ./template - - cp ./template "$secretFile" - - popd >/dev/null - - rm -rf "$TMP_DIR" + simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" "$secretFile" ''; }; };