mirror of
https://github.com/Swarsel/.dotfiles.git
synced 2025-12-06 00:57:22 +01:00
374 lines
13 KiB
Nix
374 lines
13 KiB
Nix
# inspired by https://github.com/atropos112/nixos/blob/7fef652006a1c939f4caf9c8a0cb0892d9cdfe21/modules/garage.nix
|
|
{ lib, pkgs, config, globals, dns, confLib, ... }:
|
|
let
|
|
inherit (confLib.gen {
|
|
name = "garage";
|
|
port = 3900;
|
|
domain = config.repo.secrets.common.services.domains."garage-${config.node.name}";
|
|
}) servicePort serviceName specificServiceName serviceDomain subDomain baseDomain serviceAddress serviceProxy proxyAddress4 proxyAddress6;
|
|
|
|
cfg = lib.recursiveUpdate config.services.${serviceName} config.swarselsystems.server.${serviceName};
|
|
inherit (config.swarselsystems) sopsFile mainUser;
|
|
|
|
# needs SSD
|
|
metadata_dir = "/var/lib/garage/meta";
|
|
# metadata_dir = if config.swarselsystems.isCloud then "/var/lib/garage/meta" else "/Vault/data/garage/meta";
|
|
|
|
garageRpcPort = 3901;
|
|
garageWebPort = 3902;
|
|
garageAdminPort = 3903;
|
|
garageK2VPort = 3904;
|
|
|
|
adminDomain = "${subDomain}admin.${baseDomain}";
|
|
webDomain = "${subDomain}web.${baseDomain}";
|
|
in
|
|
{
|
|
options = {
|
|
swarselmodules.server.${serviceName} = lib.mkEnableOption "enable ${serviceName} on server";
|
|
swarselsystems.server.${serviceName} = {
|
|
data_dir = {
|
|
path = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Directory where Garage stores its metadata";
|
|
};
|
|
capacity = lib.mkOption {
|
|
type = lib.types.str;
|
|
};
|
|
};
|
|
buckets = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
description = "List of buckets to create";
|
|
};
|
|
keys = lib.mkOption {
|
|
type = lib.types.attrsOf (lib.types.listOf lib.types.str);
|
|
default = { };
|
|
description = "Keys and their associated buckets. Each key gets full access (read/write/owner) to its listed buckets.";
|
|
example = {
|
|
my_key_name = [ "bucket1" "bucket2" ];
|
|
my_other_key = [ "bucket2" "bucket3" ];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
config = lib.mkIf config.swarselmodules.server.${serviceName} {
|
|
assertions = [
|
|
{
|
|
assertion = config.swarselsystems.server.${serviceName}.buckets != [ ];
|
|
message = "If Garage is enabled, at least one bucket must be specified in atro.garage.buckets";
|
|
}
|
|
{
|
|
assertion = builtins.length (lib.attrsToList config.swarselsystems.server.${serviceName}.keys) > 0;
|
|
message = "If Garage is enabled, at least one key must be specified in atro.garage.keys";
|
|
}
|
|
{
|
|
assertion =
|
|
let
|
|
allKeyBuckets = lib.flatten (lib.attrValues config.swarselsystems.server.${serviceName}.keys);
|
|
invalidBuckets = builtins.filter (bucket: !(lib.elem bucket config.swarselsystems.server.${serviceName}.buckets)) allKeyBuckets;
|
|
in
|
|
invalidBuckets == [ ];
|
|
message = "All buckets referenced in keys must exist in the buckets list";
|
|
}
|
|
];
|
|
|
|
swarselsystems.server.dns.${baseDomain}.subdomainRecords = {
|
|
"${subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
|
|
"${subDomain}admin" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
|
|
"${subDomain}web" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
|
|
"*.${subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
|
|
"*.${subDomain}web" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
|
|
};
|
|
|
|
sops = {
|
|
secrets.garage-admin-token = { inherit sopsFile; };
|
|
secrets.garage-rpc-secret = { inherit sopsFile; };
|
|
};
|
|
|
|
# DynamicUser cannot read above secrets
|
|
systemd.services.${serviceName}.serviceConfig = {
|
|
DynamicUser = false;
|
|
ProtectHome = lib.mkForce false;
|
|
};
|
|
|
|
environment = {
|
|
persistence."/persist".directories = lib.mkIf config.swarselsystems.isImpermanence [
|
|
{ directory = "/var/lib/garage"; }
|
|
(lib.mkIf config.swarselsystems.isCloud { directory = config.swarselsystems.server.${serviceName}.data_dir.path; })
|
|
];
|
|
systemPackages = [
|
|
cfg.package
|
|
];
|
|
};
|
|
|
|
globals.services.${specificServiceName} = {
|
|
domain = serviceDomain;
|
|
inherit proxyAddress4 proxyAddress6;
|
|
};
|
|
|
|
|
|
services.${serviceName} = {
|
|
enable = true;
|
|
package = pkgs.garage_2;
|
|
settings = {
|
|
data_dir = [ config.swarselsystems.server.${serviceName}.data_dir ];
|
|
inherit metadata_dir;
|
|
db_engine = "lmdb";
|
|
block_size = "128M";
|
|
use_local_tz = false;
|
|
disable_scrub = true;
|
|
replication_factor = 1;
|
|
compression_level = "none";
|
|
|
|
rpc_bind_addr = "[::]:${builtins.toString garageRpcPort}";
|
|
# we are not joining our nodes, just use the private ipv4
|
|
rpc_public_addr = "${globals.networks."${if config.swarselsystems.isCloud then config.node.name else "home"}-${config.swarselsystems.server.localNetwork}".hosts.${config.node.name}.ipv4}:${builtins.toString garageRpcPort}";
|
|
|
|
rpc_secret_file = config.sops.secrets.garage-rpc-secret.path;
|
|
|
|
s3_api = {
|
|
s3_region = mainUser;
|
|
api_bind_addr = "[::]:${builtins.toString servicePort}";
|
|
root_domain = ".${serviceDomain}";
|
|
};
|
|
|
|
s3_web = {
|
|
bind_addr = "[::]:${builtins.toString garageWebPort}";
|
|
root_domain = ".${config.repo.secrets.common.services.domains."garage-web-${config.node.name}"}";
|
|
add_host_to_metrics = true;
|
|
};
|
|
|
|
admin = {
|
|
api_bind_addr = "[::]:${builtins.toString garageAdminPort}";
|
|
admin_token_file = config.sops.secrets.garage-admin-token.path;
|
|
};
|
|
|
|
k2v_api = {
|
|
api_bind_addr = "[::]:${builtins.toString garageK2VPort}";
|
|
};
|
|
};
|
|
};
|
|
|
|
|
|
systemd.services = {
|
|
garage-buckets = {
|
|
description = "Create Garage buckets";
|
|
after = [ "garage.service" ];
|
|
wants = [ "garage.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
path = [ cfg.package pkgs.gawk pkgs.coreutils ];
|
|
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
User = "root";
|
|
Group = "root";
|
|
};
|
|
|
|
script = ''
|
|
garage status
|
|
|
|
# Checking repeatedly with garage status until getting 0 exit code
|
|
while ! garage status >/dev/null 2>&1; do
|
|
echo "Garage not yet operational, waiting..."
|
|
echo "Current garage status output:"
|
|
garage status 2>&1 || true
|
|
echo "---"
|
|
sleep 5
|
|
done
|
|
|
|
# Now we check if garage status shows any failed nodes by checking for ==== FAILED NODES ====
|
|
while garage status | grep -q "==== FAILED NODES ===="; do
|
|
echo "Garage has failed nodes, waiting..."
|
|
echo "Current garage status output:"
|
|
garage status 2>&1 || true
|
|
echo "---"
|
|
sleep 5
|
|
done
|
|
|
|
echo "Garage is operational, proceeding with bucket management."
|
|
|
|
# Get list of existing buckets
|
|
existing_buckets=$(garage bucket list | tail -n +2 | awk '{print $3}' | grep -v '^$' || true)
|
|
|
|
# Create buckets that should exist
|
|
${lib.concatMapStringsSep "\n" (bucket: ''
|
|
if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == *"Bucket not found"* ]]; then
|
|
echo "Creating bucket ${lib.escapeShellArg bucket}"
|
|
garage bucket create ${lib.escapeShellArg bucket}
|
|
else
|
|
echo "Bucket ${lib.escapeShellArg bucket} already exists"
|
|
fi
|
|
'')
|
|
cfg.buckets}
|
|
|
|
# Remove buckets that shouldn't exist
|
|
for bucket in $existing_buckets; do
|
|
should_exist=false
|
|
${lib.concatMapStringsSep "\n" (bucket: ''
|
|
if [[ "$bucket" == ${lib.escapeShellArg bucket} ]]; then
|
|
should_exist=true
|
|
fi
|
|
'')
|
|
cfg.buckets}
|
|
|
|
if [[ "$should_exist" == "false" ]]; then
|
|
echo "Removing bucket $bucket"
|
|
garage bucket delete --yes "$bucket"
|
|
fi
|
|
done
|
|
'';
|
|
};
|
|
|
|
garage-keys = {
|
|
description = "Create Garage keys and set permissions";
|
|
after = [ "garage-buckets.service" ];
|
|
wants = [ "garage-buckets.service" ];
|
|
requires = [ "garage-buckets.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
path = [ cfg.package pkgs.gawk pkgs.coreutils ];
|
|
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
User = "root";
|
|
Group = "root";
|
|
};
|
|
|
|
script = ''
|
|
garage key list
|
|
echo "Managing keys..."
|
|
|
|
# Get list of existing keys
|
|
existing_keys=$(garage key list | tail -n +2 | awk '{print $3}' | grep -v '^$' || true)
|
|
|
|
# Create keys that should exist
|
|
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: _: ''
|
|
if [[ "$(garage key info ${lib.escapeShellArg keyName} 2>&1)" == *"0 matching keys"* ]]; then
|
|
echo "Creating key ${lib.escapeShellArg keyName}"
|
|
garage key create ${lib.escapeShellArg keyName}
|
|
else
|
|
echo "Key ${lib.escapeShellArg keyName} already exists"
|
|
fi
|
|
'')
|
|
cfg.keys)}
|
|
|
|
# Set up key permissions for buckets
|
|
${lib.concatStringsSep "\n" (lib.mapAttrsToList (
|
|
keyName: buckets:
|
|
lib.concatMapStringsSep "\n" (bucket: ''
|
|
echo "Granting full access to key ${lib.escapeShellArg keyName} for bucket ${lib.escapeShellArg bucket}"
|
|
garage bucket allow --read --write --owner --key ${lib.escapeShellArg keyName} ${lib.escapeShellArg bucket}
|
|
'')
|
|
buckets
|
|
)
|
|
cfg.keys)}
|
|
|
|
# Remove permissions from buckets that are no longer associated with keys
|
|
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: buckets: ''
|
|
# Get current buckets this key has access to
|
|
current_buckets=$(garage key info ${lib.escapeShellArg keyName} | grep -A 1000 "==== BUCKETS FOR THIS KEY ====" | tail -n +3 | awk '{print $3}' | grep -v '^$' || true)
|
|
|
|
# Remove access from buckets not in the desired list
|
|
for current_bucket in $current_buckets; do
|
|
should_have_access=false
|
|
${lib.concatMapStringsSep "\n" (bucket: ''
|
|
if [[ "$current_bucket" == ${lib.escapeShellArg bucket} ]]; then
|
|
should_have_access=true
|
|
fi
|
|
'')
|
|
buckets}
|
|
|
|
if [[ "$should_have_access" == "false" ]]; then
|
|
echo "Removing access for key ${lib.escapeShellArg keyName} from bucket $current_bucket"
|
|
garage bucket deny --key ${lib.escapeShellArg keyName} $current_bucket
|
|
fi
|
|
done
|
|
'')
|
|
cfg.keys)}
|
|
|
|
# Remove keys that shouldn't exist
|
|
for key in $existing_keys; do
|
|
should_exist=false
|
|
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: _: ''
|
|
if [[ "$key" == ${lib.escapeShellArg keyName} ]]; then
|
|
should_exist=true
|
|
fi
|
|
'')
|
|
cfg.keys)}
|
|
|
|
if [[ "$should_exist" == "false" ]]; then
|
|
echo "Removing key $key"
|
|
garage key delete --yes "$key"
|
|
fi
|
|
done
|
|
'';
|
|
};
|
|
};
|
|
|
|
security.acme.certs."${webDomain}" = {
|
|
domain = "*.${webDomain}";
|
|
};
|
|
|
|
nodes.${serviceProxy}.services.nginx = {
|
|
upstreams = {
|
|
${serviceName} = {
|
|
servers = {
|
|
"${serviceAddress}:${builtins.toString servicePort}" = { };
|
|
};
|
|
};
|
|
"${serviceName}Web" = {
|
|
servers = {
|
|
"${serviceAddress}:${builtins.toString garageWebPort}" = { };
|
|
};
|
|
};
|
|
"${serviceName}Admin" = {
|
|
servers = {
|
|
"${serviceAddress}:${builtins.toString garageAdminPort}" = { };
|
|
};
|
|
};
|
|
};
|
|
virtualHosts = {
|
|
"${adminDomain}" = {
|
|
enableACME = true;
|
|
forceSSL = true;
|
|
acmeRoot = null;
|
|
oauth2.enable = false;
|
|
locations = {
|
|
"/" = {
|
|
proxyPass = "http://${serviceName}Admin";
|
|
};
|
|
};
|
|
};
|
|
"*.${webDomain}" = {
|
|
useACMEHost = webDomain;
|
|
forceSSL = true;
|
|
acmeRoot = null;
|
|
oauth2.enable = false;
|
|
locations = {
|
|
"/" = {
|
|
proxyPass = "http://${serviceName}Web";
|
|
};
|
|
};
|
|
};
|
|
"${serviceDomain}" = {
|
|
serverAliases = [ "*.${serviceDomain}" ];
|
|
enableACME = true;
|
|
forceSSL = true;
|
|
acmeRoot = null;
|
|
oauth2.enable = false;
|
|
locations = {
|
|
"/" = {
|
|
proxyPass = "http://${serviceName}";
|
|
extraConfig = ''
|
|
client_max_body_size 0;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
};
|
|
}
|