fix[server]: get cache working

This commit is contained in:
Leon Schwarzäugl 2025-11-27 03:25:34 +01:00
parent 0c3aee7997
commit 969ae3302d
Signed by: swarsel
GPG key ID: 26A54C31F2A4FD84
16 changed files with 1245 additions and 292 deletions

View file

@ -16,6 +16,7 @@ in
} // (lib.optionalAttrs (!isPublic) { });
systemd.user.sessionVariables = {
DOCUMENT_DIR_PRIV = lib.mkForce "${homeDir}/Documents/Private";
FLAKE = "${config.home.homeDirectory}/.dotfiles";
} // lib.optionalAttrs (!isPublic) {
SWARSEL_MAIL1 = address1;
SWARSEL_MAIL2 = address2;

View file

@ -25,6 +25,9 @@
# ssh login using idm
opkssh
# cache
attic-client
# dict
(aspellWithDicts (dicts: with dicts; [ de en en-computers en-science ]))

View file

@ -1,6 +1,7 @@
{ self, outputs, lib, pkgs, config, ... }:
{ self, outputs, lib, pkgs, config, globals, nixosConfig ? config, ... }:
let
inherit (config.swarselsystems) mainUser flakePath isNixos isLinux;
inherit (nixosConfig.repo.secrets.common) atticPublicKey;
in
{
options.swarselmodules.general = lib.mkEnableOption "general nix settings";
@ -33,6 +34,12 @@ in
"cgroups"
"pipe-operators"
];
substituters = [
"https://${globals.services.attic.domain}/${mainUser}"
];
trusted-public-keys = [
atticPublicKey
];
trusted-users = [ "@wheel" "${mainUser}" ];
connect-timeout = 5;
bash-prompt-prefix = "$SHLVL:\\w ";

View file

@ -1,4 +1,4 @@
{ self, lib, pkgs, config, ... }:
{ self, lib, pkgs, config, globals, ... }:
let
certsSopsFile = self + /secrets/certs/secrets.yaml;
clientSopsFile = self + /secrets/${config.node.name}/secrets.yaml;
@ -50,7 +50,7 @@ in
networking = {
inherit (config.swarselsystems) hostName;
hosts = {
"192.168.178.24" = [ "store.swarsel.win" ];
"${globals.networks.home-lan.hosts.winters.ipv4}" = [ globals.services.transmission.domain ];
};
wireless.iwd = {
enable = true;

View file

@ -1,6 +1,7 @@
{ self, lib, pkgs, config, outputs, inputs, minimal, ... }:
{ self, lib, pkgs, config, outputs, inputs, minimal, globals, ... }:
let
inherit (config.swarselsystems) mainUser;
inherit (config.repo.secrets.common) atticPublicKey;
settings = if minimal then { } else {
environment.etc."nixos/configuration.nix".source = pkgs.writeText "configuration.nix" ''
assert builtins.trace "This location is not used. The config is found in ${config.swarselsystems.flakePath}!" false;
@ -76,6 +77,12 @@ in
"cgroups"
"pipe-operators"
];
substituters = [
"https://${globals.services.attic.domain}/${mainUser}"
];
trusted-public-keys = [
atticPublicKey
];
trusted-users = [ "@wheel" "${config.swarselsystems.mainUser}" ];
};
# extraOptions = ''

View file

@ -0,0 +1,129 @@
{ lib, config, globals, dns, confLib, ... }:
let
inherit (confLib.gen { name = "attic"; port = 8091; }) serviceName serviceDir servicePort serviceAddress serviceDomain serviceProxy proxyAddress4 proxyAddress6;
inherit (config.swarselsystems) mainUser isPublic sopsFile;
serviceDB = "atticd";
in
{
options = {
swarselmodules.server.${serviceName} = lib.mkEnableOption "enable ${serviceName} on server";
};
config = lib.mkIf config.swarselmodules.server.${serviceName} {
swarselsystems.server.dns.${globals.services.${serviceName}.baseDomain}.subdomainRecords = {
"${globals.services.${serviceName}.subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
};
globals.services.${serviceName} = {
domain = serviceDomain;
inherit proxyAddress4 proxyAddress6;
};
sops = lib.mkIf (!isPublic) {
secrets = {
attic-server-token = { inherit sopsFile; };
attic-garage-access-key = { inherit sopsFile; };
attic-garage-secret-key = { inherit sopsFile; };
};
templates = {
"attic.env" = {
content = ''
ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64=${config.sops.placeholder.attic-server-token}
AWS_ACCESS_KEY_ID=${config.sops.placeholder.attic-garage-access-key}
AWS_SECRET_ACCESS_KEY=${config.sops.placeholder.attic-garage-secret-key}
'';
};
};
};
services.atticd = {
enable = true;
environmentFile = config.sops.templates."attic.env".path;
settings = {
listen = "[::]:${builtins.toString servicePort}";
api-endpoint = "https://${serviceDomain}/";
allowed-hosts = [
serviceDomain
];
require-proof-of-possession = false;
compression = {
type = "zstd";
level = 3;
};
database.url = "postgresql:///atticd?host=/run/postgresql";
storage =
if config.swarselmodules.server.garage then {
type = "s3";
region = mainUser;
bucket = serviceName;
# attic must be patched to never serve pre-signed s3 urls directly
# otherwise it will redirect clients to this localhost endpoint
endpoint = "http://127.0.0.1:3900";
} else {
type = "local";
path = serviceDir;
# attic must be patched to never serve pre-signed s3 urls directly
# otherwise it will redirect clients to this localhost endpoint
};
garbage-collection = {
interval = "1 day";
default-retention-period = "3 months";
};
chunking = {
nar-size-threshold = if config.swarselmodules.server.garage then 0 else 64 * 1024; # 64 KiB
min-size = 16 * 1024; # 16 KiB
avg-size = 64 * 1024; # 64 KiB
max-size = 256 * 1024; # 256 KiBize = 262144;
};
};
};
services.postgresql = {
enable = true;
enableTCPIP = true;
ensureDatabases = [ serviceDB ];
ensureUsers = [
{
name = serviceDB;
ensureDBOwnership = true;
}
];
};
systemd.services.atticd = lib.mkIf config.swarselmodules.server.garage {
requires = [ "garage.service" ];
after = [ "garage.service" ];
};
nodes.${serviceProxy}.services.nginx = {
upstreams = {
${serviceName} = {
servers = {
"${serviceAddress}:${builtins.toString servicePort}" = { };
};
};
};
virtualHosts = {
"${serviceDomain}" = {
enableACME = true;
forceSSL = true;
acmeRoot = null;
oauth2.enable = false;
locations = {
"/" = {
proxyPass = "http://${serviceName}";
extraConfig = ''
client_max_body_size 0;
'';
};
};
};
};
};
};
}

View file

@ -1,26 +1,82 @@
{ self, lib, pkgs, config, configName, globals, dns, confLib, ... }:
# inspired by https://github.com/atropos112/nixos/blob/7fef652006a1c939f4caf9c8a0cb0892d9cdfe21/modules/garage.nix
{ lib, pkgs, config, globals, dns, confLib, ... }:
let
inherit (confLib.gen { name = "garage"; port = 3900; }) servicePort serviceName serviceDomain serviceAddress serviceProxy proxyAddress4 proxyAddress6;
inherit (confLib.gen {
name = "garage";
port = 3900;
domain = config.repo.secrets.common.services.domains."garage-${config.node.name}";
}) servicePort serviceName specificServiceName serviceDomain subDomain baseDomain serviceAddress serviceProxy proxyAddress4 proxyAddress6;
sopsFile = self + /secrets/${configName}/secrets2.yaml;
cfg = lib.recursiveUpdate config.services.${serviceName} config.swarselsystems.server.${serviceName};
inherit (config.swarselsystems) sopsFile mainUser;
cfg = config.services.${serviceName};
# needs SSD
metadata_dir = "/var/lib/garage/meta";
# metadata_dir = if config.swarselsystems.isCloud then "/var/lib/garage/meta" else "/Vault/data/garage/meta";
garageRpcPort = 3901;
garageWebPort = 3902;
garageAdminPort = 3903;
garageK2VPort = 3904;
adminDomain = "${subDomain}admin.${baseDomain}";
webDomain = "${subDomain}web.${baseDomain}";
in
{
options = {
swarselmodules.server.${serviceName} = lib.mkEnableOption "enable ${serviceName} on server";
swarselsystems.server.${serviceName} = {
data_dir = lib.mkOption {
type = lib.types.either lib.types.path (lib.types.listOf lib.types.attrs);
default = "/var/lib/garage/data";
data_dir = {
path = lib.mkOption {
type = lib.types.str;
description = "Directory where Garage stores its metadata";
};
capacity = lib.mkOption {
type = lib.types.str;
};
};
buckets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of buckets to create";
};
keys = lib.mkOption {
type = lib.types.attrsOf (lib.types.listOf lib.types.str);
default = { };
description = "Keys and their associated buckets. Each key gets full access (read/write/owner) to its listed buckets.";
example = {
my_key_name = [ "bucket1" "bucket2" ];
my_other_key = [ "bucket2" "bucket3" ];
};
};
};
};
config = lib.mkIf config.swarselmodules.server.${serviceName} {
assertions = [
{
assertion = config.swarselsystems.server.${serviceName}.buckets != [ ];
message = "If Garage is enabled, at least one bucket must be specified in atro.garage.buckets";
}
{
assertion = builtins.length (lib.attrsToList config.swarselsystems.server.${serviceName}.keys) > 0;
message = "If Garage is enabled, at least one key must be specified in atro.garage.keys";
}
{
assertion =
let
allKeyBuckets = lib.flatten (lib.attrValues config.swarselsystems.server.${serviceName}.keys);
invalidBuckets = builtins.filter (bucket: !(lib.elem bucket config.swarselsystems.server.${serviceName}.buckets)) allKeyBuckets;
in
invalidBuckets == [ ];
message = "All buckets referenced in keys must exist in the buckets list";
}
];
swarselsystems.server.dns.${globals.services.${serviceName}.baseDomain}.subdomainRecords = {
"${globals.services.${serviceName}.subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
swarselsystems.server.dns.${baseDomain}.subdomainRecords = {
"${subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
"${subDomain}admin" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
"${subDomain}web" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
"*.${subDomain}" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
"*.${subDomain}web" = dns.lib.combinators.host proxyAddress4 proxyAddress6;
};
sops = {
@ -28,58 +84,233 @@ in
secrets.garage-rpc-secret = { inherit sopsFile; };
};
# DynamicUser cannot read above secrets
systemd.services.${serviceName}.serviceConfig = {
DynamicUser = false;
ProtectHome = lib.mkForce false;
};
environment = {
persistence."/persist".directories = lib.mkIf config.swarselsystems.isImpermanence [
{ directory = metadata_dir; }
{ directory = "/var/lib/garage"; }
(lib.mkIf config.swarselsystems.isCloud { directory = config.swarselsystems.server.${serviceName}.data_dir.path; })
];
systemPackages = [
cfg.package
];
};
globals.services.${serviceName} = {
globals.services.${specificServiceName} = {
domain = serviceDomain;
inherit proxyAddress4 proxyAddress6;
};
systemd.services.${serviceName}.serviceConfig = {
DynamicUser = false;
ProtectHome = lib.mkForce false;
};
services.${serviceName} = {
enable = true;
package = pkgs.garage_2;
settings = {
inherit (config.swarselsystems.${serviceName}) data_dir;
data_dir = [ config.swarselsystems.server.${serviceName}.data_dir ];
inherit metadata_dir;
db_engine = "lmdb";
block_size = "1MiB";
block_size = "128M";
use_local_tz = false;
disable_scrub = true;
replication_factor = 1;
compression_level = "none";
replication_factor = 2; # Number of copies of data
rpc_bind_addr = "[::]:${builtins.toString garageRpcPort}";
# we are not joining our nodes, just use the private ipv4
rpc_public_addr = "${globals.networks."${if config.swarselsystems.isCloud then config.node.name else "home"}-${config.swarselsystems.server.localNetwork}".hosts.${config.node.name}.ipv4}:${builtins.toString garageRpcPort}";
rpc_bind_addr = "[::]:3901";
rpc_public_addr = "${config.repo.secrets.local.ipv4}:4317";
rpc_secret_file = config.sops.secrets.garage-rpc-secret.path;
s3_api = {
s3_region = "swarsel";
api_bind_addr = "0.0.0.0:${builtins.toString servicePort}";
root_domain = ".s3.garage.localhost";
s3_region = mainUser;
api_bind_addr = "[::]:${builtins.toString servicePort}";
root_domain = ".${serviceDomain}";
};
s3_web = {
bind_addr = "[::]:${builtins.toString garageWebPort}";
root_domain = ".${config.repo.secrets.common.services.domains."garage-web-${config.node.name}"}";
add_host_to_metrics = true;
};
admin = {
api_bind_addr = "0.0.0.0:3903";
api_bind_addr = "[::]:${builtins.toString garageAdminPort}";
admin_token_file = config.sops.secrets.garage-admin-token.path;
};
k2v_api = {
api_bind_addr = "[::]:3904";
api_bind_addr = "[::]:${builtins.toString garageK2VPort}";
};
};
};
systemd.services = {
garage-buckets = {
description = "Create Garage buckets";
after = [ "garage.service" ];
wants = [ "garage.service" ];
wantedBy = [ "multi-user.target" ];
path = [ cfg.package pkgs.gawk pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = "root";
Group = "root";
};
script = ''
garage status
# Checking repeatedly with garage status until getting 0 exit code
while ! garage status >/dev/null 2>&1; do
echo "Garage not yet operational, waiting..."
echo "Current garage status output:"
garage status 2>&1 || true
echo "---"
sleep 5
done
# Now we check if garage status shows any failed nodes by checking for ==== FAILED NODES ====
while garage status | grep -q "==== FAILED NODES ===="; do
echo "Garage has failed nodes, waiting..."
echo "Current garage status output:"
garage status 2>&1 || true
echo "---"
sleep 5
done
echo "Garage is operational, proceeding with bucket management."
# Get list of existing buckets
existing_buckets=$(garage bucket list | tail -n +2 | awk '{print $3}' | grep -v '^$' || true)
# Create buckets that should exist
${lib.concatMapStringsSep "\n" (bucket: ''
if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == *"Bucket not found"* ]]; then
echo "Creating bucket ${lib.escapeShellArg bucket}"
garage bucket create ${lib.escapeShellArg bucket}
else
echo "Bucket ${lib.escapeShellArg bucket} already exists"
fi
'')
cfg.buckets}
# Remove buckets that shouldn't exist
for bucket in $existing_buckets; do
should_exist=false
${lib.concatMapStringsSep "\n" (bucket: ''
if [[ "$bucket" == ${lib.escapeShellArg bucket} ]]; then
should_exist=true
fi
'')
cfg.buckets}
if [[ "$should_exist" == "false" ]]; then
echo "Removing bucket $bucket"
garage bucket delete --yes "$bucket"
fi
done
'';
};
garage-keys = {
description = "Create Garage keys and set permissions";
after = [ "garage-buckets.service" ];
wants = [ "garage-buckets.service" ];
requires = [ "garage-buckets.service" ];
wantedBy = [ "multi-user.target" ];
path = [ cfg.package pkgs.gawk pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = "root";
Group = "root";
};
script = ''
garage key list
echo "Managing keys..."
# Get list of existing keys
existing_keys=$(garage key list | tail -n +2 | awk '{print $3}' | grep -v '^$' || true)
# Create keys that should exist
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: _: ''
if [[ "$(garage key info ${lib.escapeShellArg keyName} 2>&1)" == *"0 matching keys"* ]]; then
echo "Creating key ${lib.escapeShellArg keyName}"
garage key create ${lib.escapeShellArg keyName}
else
echo "Key ${lib.escapeShellArg keyName} already exists"
fi
'')
cfg.keys)}
# Set up key permissions for buckets
${lib.concatStringsSep "\n" (lib.mapAttrsToList (
keyName: buckets:
lib.concatMapStringsSep "\n" (bucket: ''
echo "Granting full access to key ${lib.escapeShellArg keyName} for bucket ${lib.escapeShellArg bucket}"
garage bucket allow --read --write --owner --key ${lib.escapeShellArg keyName} ${lib.escapeShellArg bucket}
'')
buckets
)
cfg.keys)}
# Remove permissions from buckets that are no longer associated with keys
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: buckets: ''
# Get current buckets this key has access to
current_buckets=$(garage key info ${lib.escapeShellArg keyName} | grep -A 1000 "==== BUCKETS FOR THIS KEY ====" | tail -n +3 | awk '{print $3}' | grep -v '^$' || true)
# Remove access from buckets not in the desired list
for current_bucket in $current_buckets; do
should_have_access=false
${lib.concatMapStringsSep "\n" (bucket: ''
if [[ "$current_bucket" == ${lib.escapeShellArg bucket} ]]; then
should_have_access=true
fi
'')
buckets}
if [[ "$should_have_access" == "false" ]]; then
echo "Removing access for key ${lib.escapeShellArg keyName} from bucket $current_bucket"
garage bucket deny --key ${lib.escapeShellArg keyName} $current_bucket
fi
done
'')
cfg.keys)}
# Remove keys that shouldn't exist
for key in $existing_keys; do
should_exist=false
${lib.concatStringsSep "\n" (lib.mapAttrsToList (keyName: _: ''
if [[ "$key" == ${lib.escapeShellArg keyName} ]]; then
should_exist=true
fi
'')
cfg.keys)}
if [[ "$should_exist" == "false" ]]; then
echo "Removing key $key"
garage key delete --yes "$key"
fi
done
'';
};
};
security.acme.certs."${webDomain}" = {
domain = "*.${webDomain}";
};
nodes.${serviceProxy}.services.nginx = {
upstreams = {
${serviceName} = {
@ -87,9 +318,42 @@ in
"${serviceAddress}:${builtins.toString servicePort}" = { };
};
};
"${serviceName}Web" = {
servers = {
"${serviceAddress}:${builtins.toString garageWebPort}" = { };
};
};
"${serviceName}Admin" = {
servers = {
"${serviceAddress}:${builtins.toString garageAdminPort}" = { };
};
};
};
virtualHosts = {
"${adminDomain}" = {
enableACME = true;
forceSSL = true;
acmeRoot = null;
oauth2.enable = false;
locations = {
"/" = {
proxyPass = "http://${serviceName}Admin";
};
};
};
"*.${webDomain}" = {
useACMEHost = webDomain;
forceSSL = true;
acmeRoot = null;
oauth2.enable = false;
locations = {
"/" = {
proxyPass = "http://${serviceName}Web";
};
};
};
"${serviceDomain}" = {
serverAliases = [ "*.${serviceDomain}" ];
enableACME = true;
forceSSL = true;
acmeRoot = null;
@ -97,6 +361,9 @@ in
locations = {
"/" = {
proxyPass = "http://${serviceName}";
extraConfig = ''
client_max_body_size 0;
'';
};
};
};

View file

@ -2,6 +2,7 @@
let
inherit (confLib.gen { name = "postgresql"; port = 3254; }) serviceName;
postgresVersion = 14;
postgresDirPrefix = if config.swarselsystems.isCloud then "/var/lib" else "/Vault/data";
in
{
options.swarselmodules.server.${serviceName} = lib.mkEnableOption "enable ${serviceName} on server";
@ -10,8 +11,12 @@ in
${serviceName} = {
enable = true;
package = pkgs."postgresql_${builtins.toString postgresVersion}";
dataDir = "/Vault/data/${serviceName}/${builtins.toString postgresVersion}";
dataDir = "${postgresDirPrefix}/${serviceName}/${builtins.toString postgresVersion}";
};
};
environment.persistence."/persist".directories = lib.mkIf (config.swarselsystems.isImpermanence && config.swarselsystems.isCloud) [
{ directory = "/var/lib/postgresql"; user = "postgres"; group = "postgres"; mode = "0750"; }
];
};
}

View file

@ -1,4 +1,4 @@
{ config, globals, ... }:
{ config, lib, globals, ... }:
{
_module.args = {
confLib = rec {
@ -11,9 +11,12 @@
gen = { name, user ? name, group ? name, dir ? null, port ? null, domain ? (domainDefault name), address ? addressDefault, proxy ? proxyDefault }: rec {
servicePort = port;
serviceName = name;
specificServiceName = "${name}-${config.node.name}";
serviceUser = user;
serviceGroup = group;
serviceDomain = domain;
baseDomain = lib.swarselsystems.getBaseDomain domain;
subDomain = lib.swarselsystems.getSubDomain domain;
serviceDir = dir;
serviceAddress = address;
serviceProxy = proxy;