chaos/nixfiles
1
0
Fork 0
Code Issues 2 Releases Activity

update

Browse source
This commit is contained in:
chaos 2024-11-08 20:50:43 +00:00
parent 7891d24456
commit 71375e0414
9 changed files with 23 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
hosts/hetzner-arm/containers/storage/data/rclone_config.template
View file

@ -1,74 +1,15 @@
[Storage]
type = combine
upstreams = "Backups=B2-Chaos-Backups:Chaos-Backups" "Photos=B2-Chaos-Photos:Chaos-Photos" "Music=B2-Chaos-Music:Chaos-Music" "Public=B2-Chaos-Public:Chaos-Public" "Notes=Notes:" "Media=Media:" "Personal=B2-Chaos-Personal:Chaos-Personal" "Uploads=B2-Chaos-Uploads:Chaos-Uploads"
upstreams = "Backups=B2:Chaos-Backups" "Photos=B2:Chaos-Photos" "Music=B2:Chaos-Music" "Public=B2:Chaos-Public" "Notes=Notes:" "Personal=B2:Chaos-Personal" "Uploads=B2:Chaos-Uploads"
[B2-Chaos-Backups]
[B2]
type = b2
account = B2_CHAOS_BACKUPS_ACCOUNT
key = B2_CHAOS_BACKUPS_KEY
account = B2_ACCOUNT
key = B2_KEY
hard_delete = true
[B2-Chaos-Photos]
type = b2
account = B2_CHAOS_PHOTOS_ACCOUNT
key = B2_CHAOS_PHOTOS_KEY
hard_delete = true
[B2-Chaos-Music]
type = b2
account = B2_CHAOS_MUSIC_ACCOUNT
key = B2_CHAOS_MUSIC_KEY
hard_delete = true
[B2-Chaos-Personal]
type = b2
account = B2_CHAOS_PERSONAL_ACCOUNT
key = B2_CHAOS_PERSONAL_KEY
hard_delete = true
[B2-Chaos-Public]
type = b2
account = B2_CHAOS_PUBLIC_ACCOUNT
key = B2_CHAOS_PUBLIC_KEY
hard_delete = true
[B2-Chaos-Notes]
type = b2
account = B2_CHAOS_NOTES_ACCOUNT
key = B2_CHAOS_NOTES_KEY
hard_delete = true
[B2-Chaos-Media]
type = b2
account = B2_CHAOS_MEDIA_ACCOUNT
key = B2_CHAOS_MEDIA_KEY
hard_delete = true
[B2-Chaos-Uploads]
type = b2
account = B2_CHAOS_UPLOADS_ACCOUNT
key = B2_CHAOS_UPLOADS_KEY
hard_delete = true
[B2-Phoenix-Cryptidz-Storage]
type = b2
account = B2_PHOENIX_CRYPTIDZ_STORAGE_ACCOUNT
key = B2_PHOENIX_CRYPTIDZ_STORAGE_KEY
hard_delete = true
[Media-Crypt]
type = crypt
remote = B2-Chaos-Media:Chaos-Media
password = STORAGE_MEDIA_CRYPT_PASSWORD
password2 = STORAGE_MEDIA_CRYPT_SALT
[Media]
type = chunker
remote = Media-Crypt:
chunk_size = 256Mi
[Notes]
type = crypt
remote = B2-Chaos-Notes:Chaos-Notes
remote = B2:Chaos-Notes
password = STORAGE_NOTES_CRYPT_PASSWORD
password2 = STORAGE_NOTES_CRYPT_SALT

24
hosts/hetzner-arm/containers/storage/profiles/rcloneServe.nix
View file

@ -15,18 +15,18 @@ in {
user = "storage";
}
// remote) [
{
id = "main";
remote = "Storage:";
type = "webdav";
extraArgs = [
"--addr=0.0.0.0:${toString ports.webdav_main}"
"--htpasswd=${secrets.webdav_main_htpasswd.path}"
"--baseurl=/Main/"
"--cache-dir=/caches/main_webdav_serve"
"--vfs-cache-mode=full"
];
}
#{
# id = "main";
# remote = "Storage:";
# type = "webdav";
# extraArgs = [
# "--addr=0.0.0.0:${toString ports.webdav_main}"
# "--htpasswd=${secrets.webdav_main_htpasswd.path}"
# "--baseurl=/Main/"
# "--cache-dir=/caches/main_webdav_serve"
# "--vfs-cache-mode=full"
# ];
#}
{
id = "music-ro";
remote = "Storage:Music";

53
hosts/hetzner-arm/containers/storage/profiles/rcloneSync.nix
View file

@ -1,53 +0,0 @@
{self, ...}: let
backupSchedules = import "${self}/data/backupSchedules.nix";
in {
services.rclone-sync = {
enable = true;
user = "storage";
syncJobs = map (syncJob:
syncJob
// {
timerConfig = backupSchedules.remoteBackups;
extraArgs = [
"--fast-list"
"--check-first"
"--delete-before"
"--b2-upload-concurrency=4"
"--transfers=4"
"--bwlimit 80M"
];
}) [
# Pheonix System's B2
{
source = "Storage:Backups";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Backups";
id = "phoenix_b2_backups";
}
{
source = "Storage:Photos";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Photos";
id = "phoenix_b2_photos";
}
{
source = "Storage:Music";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Music";
id = "phoenix_b2_music";
}
{
source = "Storage:Personal";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Personal";
id = "phoenix_b2_personal";
}
{
source = "Storage:Public";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Public";
id = "phoenix_b2_public";
}
{
source = "B2-Chaos-Notes:Chaos-Notes";
dest = "B2-Phoenix-Cryptidz-Storage:Phoenix-Cryptidz-Storage/Notes";
id = "phoenix_b2_notes";
}
];
};
}

24
hosts/hetzner-arm/containers/storage/secrets.nix
View file

@ -8,22 +8,13 @@
};
requiredVaultPaths = [
"api-keys/data/backblaze/Chaos-Backups"
"api-keys/data/backblaze/Chaos-Photos"
"api-keys/data/backblaze/Chaos-Music"
"api-keys/data/backblaze/Chaos-Personal"
"api-keys/data/backblaze/Chaos-Public"
"api-keys/data/backblaze/Chaos-Media"
"api-keys/data/backblaze/Chaos-Notes"
"api-keys/data/backblaze/Chaos-Uploads"
"api-keys/data/backblaze/Phoenix-Cryptidz-Storage"
"api-keys/data/backblaze/Backblaze"
"api-keys/data/storage/webdav/Main"
#"api-keys/data/storage/webdav/Main"
"api-keys/data/storage/webdav/Public"
"api-keys/data/storage/webdav/Uploads"
"api-keys/data/storage/webdav/Notes"
"private-public-keys/data/rclone/Chaos-Media-Crypt"
"private-public-keys/data/rclone/Chaos-Notes-Crypt"
];
@ -113,17 +104,8 @@
fetchScript = ''
cp ${./data/rclone_config.template} "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Media" "B2_CHAOS_MEDIA" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Backups" "B2_CHAOS_BACKUPS" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Photos" "B2_CHAOS_PHOTOS" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Music" "B2_CHAOS_MUSIC" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Personal" "B2_CHAOS_PERSONAL" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Public" "B2_CHAOS_PUBLIC" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Notes" "B2_CHAOS_NOTES" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Chaos-Uploads" "B2_CHAOS_UPLOADS" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Phoenix-Cryptidz-Storage" "B2_PHOENIX_CRYPTIDZ_STORAGE" "$secretFile"
simple_get_replace_b2 "/api-keys/backblaze/Backblaze" "B2" "$secretFile"
simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Media-Crypt" "STORAGE_MEDIA_CRYPT" "$secretFile"
simple_get_replace_crypt "/private-public-keys/rclone/Chaos-Notes-Crypt" "STORAGE_NOTES_CRYPT" "$secretFile"
'';
};

1
hosts/hetzner-arm/containers/storage/storage.nix
View file

@ -58,7 +58,6 @@ in {
(with hosts.hetzner-arm.containers.storage.profiles; [
rcloneConfigs
rcloneServe
rcloneSync
users
])

6
hosts/hetzner-arm/secrets.nix
View file

@ -15,7 +15,7 @@
"private-public-keys/data/ssh/root@hetzner-arm"
"private-public-keys/data/ssh/root@hetzner-arm-decrypt"
"api-keys/data/backblaze/Chaos-Backups"
"api-keys/data/backblaze/Backblaze"
"private-public-keys/data/restic/Social"
"api-keys/data/chaos_mail/gotosocial"
@ -68,8 +68,8 @@
restic_backups_env = {
fetchScript = ''
cat << EOF > "$secretFile"
AWS_ACCESS_KEY_ID=$(simple_get "/api-keys/backblaze/Chaos-Backups" .keyID)
AWS_SECRET_ACCESS_KEY=$(simple_get "/api-keys/backblaze/Chaos-Backups" .applicationKey)
AWS_ACCESS_KEY_ID=$(simple_get "/api-keys/backblaze/Backblaze" .keyID)
AWS_SECRET_ACCESS_KEY=$(simple_get "/api-keys/backblaze/Backblaze" .applicationKey)
EOF
'';
};

1
hosts/nixos.nix
View file

@ -30,7 +30,6 @@
inputs.vaultui.nixosModules.default
tree.modules.nixos.rcloneServe
tree.modules.nixos.rcloneSync
tree.modules.nixos.secrets
tree.modules.nixos.encryptedDrive
];

129
modules/nixos/rcloneSync.nix
View file

@ -1,129 +0,0 @@
{
config,
lib,
pkgs,
...
}: let
inherit (lib.modules) mkIf mkMerge;
inherit (lib.options) mkOption;
inherit (lib.strings) concatStringsSep;
inherit (lib) types;
inherit (builtins) listToAttrs;
cfg = config.services.rclone-sync;
daemonService = syncConfig: {
serviceConfig = mkMerge [
{
Type = "oneshot";
User =
if cfg.user != null
then "${cfg.user}"
else "root";
ExecStart = "${pkgs.rclone}/bin/rclone sync ${syncConfig.source} ${syncConfig.dest} ${concatStringsSep " " syncConfig.extraArgs} -P";
}
(mkIf syncConfig.autoRestart {
TimeoutSec = 60;
Restart = "on-failure";
})
syncConfig.serviceConfig
];
};
in {
options = {
services.rclone-sync = {
enable = mkOption {
type = types.bool;
default = false;
};
user = mkOption {
type = types.str;
default = null;
};
syncJobs = mkOption {
type = types.listOf (types.submodule {
options = {
source = mkOption {type = types.str;};
dest = mkOption {type = types.str;};
id = mkOption {type = types.str;};
extraArgs = mkOption {
type = types.listOf types.str;
default = [];
};
autoRestart = mkOption {
type = types.bool;
default = true;
};
timerConfig = mkOption {
type = types.attrs;
default = {
OnStartupSec = "1m";
OnUnitActiveSec = "2h";
};
};
serviceConfig = mkOption {
type = types.attrs;
default = {};
};
};
});
default = [];
};
};
};
config = mkMerge [
(mkIf (cfg.enable && cfg.syncJobs != []) {
environment.systemPackages =
[
(pkgs.writeShellScriptBin "rclone-sync-stop-all" (concatStringsSep "\n" (map (
job: ''
systemctl stop rclone-sync-${job.id}.service
''
)
cfg.syncJobs)))
(pkgs.writeShellScriptBin "rclone-sync-all" (concatStringsSep "\n" (map (
job: ''
${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} ${concatStringsSep " " job.extraArgs} -P $@
''
)
cfg.syncJobs)))
]
++ (
map (
job:
pkgs.writeShellScriptBin "rclone-manual-sync-${job.id}" ''
exec ${pkgs.rclone}/bin/rclone sync ${job.source} ${job.dest} ${concatStringsSep " " job.extraArgs} -P $@
''
)
cfg.syncJobs
);
systemd.services = listToAttrs (map (job: {
name = "rclone-sync-${job.id}";
value = daemonService job;
})
cfg.syncJobs);
systemd.timers = listToAttrs (map (job: let
name = "rclone-sync-${job.id}";
in {
inherit name;
value = {
wantedBy = ["timers.target"];
partOf = ["${name}.service"];
inherit (job) timerConfig;
};
})
cfg.syncJobs);
})
];
}

1
presets/nixos/containerBase.nix
View file

@ -11,7 +11,6 @@
presets.home-manager.by-user.root.minimalServer
modules.nixos.rcloneServe
modules.nixos.rcloneSync
modules.nixos.secrets
])
++ [