From 4508af478767c77a748583b7e05b786bda0a764b Mon Sep 17 00:00:00 2001 From: Chaos Date: Sun, 20 Nov 2022 10:34:55 +0000 Subject: [PATCH] storage server for media & updates --- flake.lock | 24 ++++++++++++------------ hosts/storage/ports.nix | 14 +++++++------- hosts/storage/profiles/rclone-serve.nix | 21 +++++++++++++++++++++ hosts/storage/rclone_config.template | 13 ++++++++++++- hosts/storage/secrets.nix | 14 ++++++++++++++ hosts/storage/storage.nix | 3 --- overlay/invidious/versions.json | 4 ++-- 7 files changed, 68 insertions(+), 25 deletions(-) diff --git a/flake.lock b/flake.lock index 0ce8474..320bafc 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "utils": "utils" }, "locked": { - "lastModified": 1668453806, - "narHash": "sha256-rDyF0essyFdCIo336gI6nPjWhjoczGn701D1JID5wl8=", + "lastModified": 1668797197, + "narHash": "sha256-0w6iD3GSSQbIeSFVDzAAQZB+hDq670ZTms3d9XI+BtM=", "owner": "serokell", "repo": "deploy-rs", - "rev": "be40823735bbdc40c1f6b7725c8b74d5a85d8023", + "rev": "2a3c5f70eee04a465aa534d8bd4fcc9bb3c4a8ce", "type": "github" }, "original": { @@ -100,11 +100,11 @@ "utils": "utils_3" }, "locked": { - "lastModified": 1668332334, - "narHash": "sha256-YT1qcE/MCqBO1Bi/Yr6GcFpNKsvmzrBKh8juyXDbxQc=", + "lastModified": 1668900402, + "narHash": "sha256-IhVlueHoQNoN0SOHZIceKU3LyEL00g2ei0aUlaNypbQ=", "owner": "nix-community", "repo": "home-manager", - "rev": "bc90de24d898655542589237cc0a6ada7564cb6c", + "rev": "c0f9cbcf93ca22e4f0ca66843be61a4bdf6f0a44", "type": "github" }, "original": { @@ -137,11 +137,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1668596599, - "narHash": "sha256-rhHyZTGI31/OfgYa9xF49UTchDXTI94pEsSNa0fOkpk=", + "lastModified": 1668765800, + "narHash": "sha256-rC40+/W6Hio7b/RsY8SvQPKNx4WqNcTgfYv8cUMAvJk=", "owner": "nixos", "repo": "nixpkgs", - "rev": "b457130e8a21608675ddf12c7d85227b22a27112", + "rev": "52b2ac8ae18bbad4374ff0dd5aeee0fdf1aea739", "type": "github" }, "original": { @@ -153,11 +153,11 @@ }, "nur": { "locked": { - "lastModified": 1668680510, - "narHash": "sha256-5O/8cXku2/JxY3AglxnzWRb3I8kFZSQRh3YHLeegYA8=", + "lastModified": 1668923022, + "narHash": "sha256-95GW/QXMczzMZ0wSz/rRGQwi2nx5BVi0qSI6aGG4OrY=", "owner": "nix-community", "repo": "NUR", - "rev": "6970572e6143893eab37a8aacfe9dd872d48a867", + "rev": "43fd9acff9fe06264ff2c045ec95cb3078c80352", "type": "github" }, "original": { diff --git a/hosts/storage/ports.nix b/hosts/storage/ports.nix index 6708627..e2d68b8 100644 --- a/hosts/storage/ports.nix +++ b/hosts/storage/ports.nix @@ -1,12 +1,12 @@ { }: { rclone_serve_webdav_main = 4242; - rclone_serve_webdav_music_ro = 4243; + rclone_serve_webdav_media = 4243; + rclone_serve_webdav_music_ro = 4244; - rclone_serve_restic_hvm = 4244; - rclone_serve_restic_music = 4245; - rclone_serve_restic_vault = 4246; - - rclone_serve_http_music = 4247; - rclone_serve_http_public = 4248; + rclone_serve_restic_hvm = 4245; + rclone_serve_restic_music = 4246; + rclone_serve_restic_vault = 4247; + rclone_serve_http_music = 4248; + rclone_serve_http_public = 4249; } diff --git a/hosts/storage/profiles/rclone-serve.nix b/hosts/storage/profiles/rclone-serve.nix index 09e4ead..04cb6ec 100644 --- a/hosts/storage/profiles/rclone-serve.nix +++ b/hosts/storage/profiles/rclone-serve.nix @@ -3,6 +3,12 @@ let secrets = config.services.secrets.secrets; ports = (import ../ports.nix { }); in { + systemd.tmpfiles.rules = [ + "d /caches - storage storage" + "d /caches/main_webdav_serve - storage storage" + "d /caches/media_webdav_serve - storage storage" + ]; + services.rclone-serve = let serviceConfig = { after = [ "secrets-init.service" ]; @@ -24,6 +30,19 @@ in { ]; inherit serviceConfig; } + { + user = "storage"; + remote = "Media-Combine-Serve:"; + type = "webdav"; + extraArgs = [ + "--addr=:${toString ports.rclone_serve_webdav_media}" + "--htpasswd=${secrets.webdav_media_htpasswd.path}" + "--baseurl=/media/" + "--cache-dir=/caches/media_webdav_serve" + "--vfs-cache-mode=full" + ]; + inherit serviceConfig; + } { user = "storage"; remote = "StorageBox:Music"; @@ -101,6 +120,8 @@ in { locations = { "/main/".proxyPass = "http://localhost:${toString ports.rclone_serve_webdav_main}"; + "/media/".proxyPass = + "http://localhost:${toString ports.rclone_serve_webdav_media}"; "/music_ro/".proxyPass = "http://localhost:${toString ports.rclone_serve_webdav_music_ro}"; }; diff --git a/hosts/storage/rclone_config.template b/hosts/storage/rclone_config.template index c434ab0..1d7ec7d 100644 --- a/hosts/storage/rclone_config.template +++ b/hosts/storage/rclone_config.template @@ -59,4 +59,15 @@ hard_delete = true [B2-Phoenix-Cryptidz-Storage] type = alias -remote = B2-Phoenix-Cryptidz-Storage-Source: \ No newline at end of file +remote = B2-Phoenix-Cryptidz-Storage-Source: + +[PutIO-WebDAV] +type = webdav +url = https://webdav.put.io +vendor = nextcloud +user = chaoticryptidz +pass = PUTIO_PASSWORD + +[Media-Combine-Serve] +type = combine +upstreams = "Media=StorageBox:Media" "PutIO=PutIO-WebDAV:" \ No newline at end of file diff --git a/hosts/storage/secrets.nix b/hosts/storage/secrets.nix index e5c0ac5..5761710 100644 --- a/hosts/storage/secrets.nix +++ b/hosts/storage/secrets.nix @@ -67,6 +67,16 @@ ''; }; + webdav_media_htpasswd = { + user = "storage"; + group = "storage"; + fetchScript = '' + username=$(simple_get "/api-keys/storage/webdav/media" .username) + password=$(simple_get "/api-keys/storage/webdav/media" .password) + htpasswd -bc "$secretFile" "$username" "$password" 2>&1 + ''; + }; + rclone_config = { user = "storage"; group = "storage"; @@ -85,6 +95,10 @@ simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" ./template simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" ./template + PUTIO_PASSWORD="token/$(simple_get /api-keys/putio .oauth_token)" + PUTIO_PASSWORD="$(rclone obscure "$PUTIO_PASSWORD")" + sed -i "s/PUTIO_PASSWORD/$PUTIO_PASSWORD/" ./template + cp ./template $secretFile popd diff --git a/hosts/storage/storage.nix b/hosts/storage/storage.nix index 775f17c..e9657a3 100644 --- a/hosts/storage/storage.nix +++ b/hosts/storage/storage.nix @@ -22,9 +22,6 @@ in { ]; systemd.tmpfiles.rules = [ - "d /caches - storage storage" - "d /caches/main_webdav_serve - storage storage" - "d /root/.config - root root" "d /root/.config/rclone - root root" "L /root/.config/rclone/rclone.conf - - - - ${secrets.rclone_config.path}" diff --git a/overlay/invidious/versions.json b/overlay/invidious/versions.json index 0381f66..009e73b 100644 --- a/overlay/invidious/versions.json +++ b/overlay/invidious/versions.json @@ -4,9 +4,9 @@ "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" }, "invidious": { - "rev": "47cc26cb3c5862e6ae96f89882ee08c6a8185672", + "rev": "516efd2df3f7d242c2d1df416053b4991a554116", "sha256": "sha256-0Cb1Qsn6vnrzd4pZm1GZxlVQNn5dYKUR/xWMCG37GSk=", - "version": "unstable-2022-11-16" + "version": "unstable-2022-11-17" }, "lsquic": { "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=",