chaos/nixfiles
1
0
Fork 0
Code Issues 2 Releases Activity

move media storage to b2, fix gotosocial media prune

Browse source
This commit is contained in:
chaos 2023-11-13 17:49:31 +00:00
parent e004cb0d5a
commit b59686aa56
No known key found for this signature in database
8 changed files with 117 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
hosts/hetzner-arm/containers/jellyfin/data/rclone_config.template
View file

@ -1,6 +1,31 @@
[Media] [PutIO-WebDAV]
type = webdav type = webdav
url = https://storage-webdav.owo.monster/Media/ url = https://webdav.put.io
vendor = other vendor = other
user = media user = chaoticryptidz
pass = MEDIA_PASSWORD pass = PUTIO_PASSWORD
[B2-Media-Source]
type = b2
account = B2_CHAOS_MEDIA_ACCOUNT
key = B2_CHAOS_MEDIA_KEY
hard_delete = true
[B2-Media]
type = alias
remote = B2-Media-Source:Chaos-Media
[Media-Source]
type = crypt
remote = B2-Media:
password = STORAGE_MEDIA_CRYPT_PASSWORD
password2 = STORAGE_MEDIA_CRYPT_SALT
[Media]
type = chunker
remote = Media-Source:
chunk_size = 256Mi
[Media-Combine]
type = combine
upstreams = "Media=Media:" "PutIO=PutIO-WebDAV:"

6
hosts/hetzner-arm/containers/jellyfin/profiles/mediaMount.nix
View file

@ -8,12 +8,16 @@
${pkgs.rclone}/bin/rclone --config ${secrets.rclone_config.path} "$@" ${pkgs.rclone}/bin/rclone --config ${secrets.rclone_config.path} "$@"
''; '';
mountMedia = pkgs.writeShellScriptBin "mount-media" '' mountMedia = pkgs.writeShellScriptBin "mount-media" ''
${rcloneMedia}/bin/rclone-media mount Media: /Media \ ${rcloneMedia}/bin/rclone-media mount Media-Combine: /Media \
--allow-other \ --allow-other \
--uid=${toString config.users.users.jellyfin.uid} \ --uid=${toString config.users.users.jellyfin.uid} \
--gid=${toString config.users.groups.jellyfin.gid} \ --gid=${toString config.users.groups.jellyfin.gid} \
--fast-list \ --fast-list \
--umask=666 \ --umask=666 \
--dir-cache-time=1m \
--vfs-cache-mode=full \
--vfs-cache-max-age=120m \
--vfs-cache-max-size=5g \
--log-level=INFO "$@" --log-level=INFO "$@"
''; '';
in { in {

50
hosts/hetzner-arm/containers/jellyfin/secrets.nix
View file

@ -3,9 +3,42 @@
enable = true; enable = true;
packages = with pkgs; [ packages = with pkgs; [
# for rclone obscure config file
rclone rclone
]; ];
extraFunctions = ''
replace_slash_for_sed() {
sed "s#/#\\\/#"
}
simple_get_obscure() {
rclone obscure "$(simple_get "$@")"
}
simple_get_replace_b2() {
api_account=$(simple_get "$1" .keyID | replace_slash_for_sed)
api_key=$(simple_get "$1" .applicationKey | replace_slash_for_sed)
replace_account=''${2}_ACCOUNT
replace_key=''${2}_KEY
sed -i "s/$replace_account/$api_account/" "$3"
sed -i "s/$replace_key/$api_key/" "$3"
}
simple_get_replace_crypt() {
password=$(simple_get_obscure "$1" .password)
salt=$(simple_get_obscure "$1" .salt)
replace_password=''${2}_PASSWORD
replace_salt=''${2}_SALT
sed -i "s/$replace_password/$password/" "$3"
sed -i "s/$replace_salt/$salt/" "$3"
}
'';
vaultLogin = { vaultLogin = {
enable = true; enable = true;
loginUsername = "hetzner-arm-container-jellyfin"; loginUsername = "hetzner-arm-container-jellyfin";
@ -16,9 +49,11 @@
}; };
requiredVaultPaths = [ requiredVaultPaths = [
"api-keys/data/storage/webdav/Media" "api-keys/data/backblaze/Chaos-Media"
"api-keys/data/storage/restic/Jellyfin" "api-keys/data/putio"
"private-public-keys/data/rclone/Chaos-Media-Crypt"
"api-keys/data/storage/restic/Jellyfin"
"private-public-keys/data/restic/Jellyfin" "private-public-keys/data/restic/Jellyfin"
]; ];
@ -32,9 +67,14 @@
group = "jellyfin"; group = "jellyfin";
fetchScript = '' fetchScript = ''
cp ${./data/rclone_config.template} "$secretFile" cp ${./data/rclone_config.template} "$secretFile"
MEDIA_PASSWORD="$(simple_get "/api-keys/storage/webdav/Media" .media)"
MEDIA_PASSWORD="$(rclone obscure "$MEDIA_PASSWORD")" simple_get_replace_b2 "/api-keys/backblaze/Chaos-Media" "B2_CHAOS_MEDIA" "$secretFile"
sed -i "s/MEDIA_PASSWORD/$MEDIA_PASSWORD/" "$secretFile"
PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)"
PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")"
sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" "$secretFile"
simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" "$secretFile"
''; '';
}; };

6
hosts/hetzner-arm/containers/social/profiles/gotosocial.nix
View file

@ -41,11 +41,11 @@ in {
smtp-host = "mail.owo.monster"; smtp-host = "mail.owo.monster";
smtp-port = 587; smtp-port = 587;
smtp-from = "gotosocial@owo.monster";
smtp-username = "gotosocial@owo.monster"; smtp-username = "gotosocial@owo.monster";
smtp-password = ""; # set via env variables smtp-password = ""; # set via env variables
smtp-from = "gotosocial@owo.monster";
media-remote-cache-days = 2; media-remote-cache-days = 1;
media-image-max-size = 1000000 * 64; # MB media-image-max-size = 1000000 * 64; # MB
media-video-max-size = 1000000 * 1024; # MB media-video-max-size = 1000000 * 1024; # MB
@ -64,7 +64,7 @@ in {
# instance-inject-mastodon-version = true; # instance-inject-mastodon-version = true;
cache = { cache = {
memory-target = "256MiB"; memory-target = "512MiB";
}; };
}; };
}; };

2
hosts/hetzner-arm/containers/social/profiles/restic.nix
View file

@ -23,7 +23,7 @@
${goToSocialAdmin} export --path /var/lib/gotosocial/gts-export.json ${goToSocialAdmin} export --path /var/lib/gotosocial/gts-export.json
${goToSocialAdmin} media prune all ${goToSocialAdmin} media prune all --dry-run=false
systemctl start gotosocial systemctl start gotosocial
'') '')

4
hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix
View file

@ -33,8 +33,10 @@ in {
]; ];
} }
{ {
# This isn't really being directly used anymore but its nice as a backup
# Jellyfin directly mounts from StorageBox instead
id = "media-combine"; id = "media-combine";
remote = "Media-Combine-Serve:"; remote = "Media-Combine:";
type = "webdav"; type = "webdav";
extraArgs = [ extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav_media}" "--addr=0.0.0.0:${toString ports.webdav_media}"

23
hosts/hetzner-arm/containers/storage/rclone_config.template
View file

@ -99,12 +99,27 @@ vendor = other
user = chaoticryptidz user = chaoticryptidz
pass = PUTIO_PASSWORD pass = PUTIO_PASSWORD
[Storage-Media-Crypt] [B2-Media-Source]
type = b2
account = B2_CHAOS_MEDIA_ACCOUNT
key = B2_CHAOS_MEDIA_KEY
hard_delete = true
[B2-Media]
type = alias
remote = B2-Media-Source:Chaos-Media
[Media-Source]
type = crypt type = crypt
remote = StorageBox:Media remote = B2-Media:
password = STORAGE_MEDIA_CRYPT_PASSWORD password = STORAGE_MEDIA_CRYPT_PASSWORD
password2 = STORAGE_MEDIA_CRYPT_SALT password2 = STORAGE_MEDIA_CRYPT_SALT
[Media-Combine-Serve] [Media]
type = chunker
remote = Media-Source:
chunk_size = 256Mi
[Media-Combine]
type = combine type = combine
upstreams = "Media=Storage-Media-Crypt:" "PutIO=PutIO-WebDAV:" upstreams = "Media=Media:" "PutIO=PutIO-WebDAV:"

32
hosts/hetzner-arm/containers/storage/secrets.nix
View file

@ -21,6 +21,7 @@
"api-keys/data/backblaze/Chaos-Music" "api-keys/data/backblaze/Chaos-Music"
"api-keys/data/backblaze/Chaos-Personal" "api-keys/data/backblaze/Chaos-Personal"
"api-keys/data/backblaze/Chaos-Public" "api-keys/data/backblaze/Chaos-Public"
"api-keys/data/backblaze/Chaos-Media"
"api-keys/data/backblaze/Phoenix-Cryptidz-Storage" "api-keys/data/backblaze/Phoenix-Cryptidz-Storage"
"api-keys/data/storage/restic/Music" "api-keys/data/storage/restic/Music"
@ -211,33 +212,24 @@
user = "storage"; user = "storage";
group = "storage"; group = "storage";
fetchScript = '' fetchScript = ''
TMP_DIR="$(mktemp -d)" cp ${./rclone_config.template} "$secretFile"
cp ${./rclone_config.template} "$TMP_DIR/template"
pushd "$TMP_DIR" >/dev/null
STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password) STORAGEBOX_PASSWORD=$(simple_get_obscure /api-keys/hetzner/storagebox .password)
sed -i "s/STORAGEBOX_PASSWORD/$STORAGEBOX_PASSWORD/" ./template sed -i "s/STORAGEBOX_PASSWORD/$STORAGEBOX_PASSWORD/" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Backups" "B2_CHAOS_BACKUPS" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Media" "B2_CHAOS_MEDIA" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Photos" "B2_CHAOS_PHOTOS" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Backups" "B2_CHAOS_BACKUPS" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Photos" "B2_CHAOS_PHOTOS" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Personal" "B2_CHAOS_PERSONAL" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Public" "B2_CHAOS_PUBLIC" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Personal" "B2_CHAOS_PERSONAL" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" ./template simple_get_replace_b2 "/api-keys/backblaze/Chaos-Public" "B2_CHAOS_PUBLIC" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" "$secretFile"
PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)" PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)"
PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")" PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")"
sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" ./template sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" "$secretFile"
simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" ./template simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" "$secretFile"
cp ./template "$secretFile"
popd >/dev/null
rm -rf "$TMP_DIR"
''; '';
}; };
}; };