From 8803e130cc33187026fe6fd4e0680cca6d65d9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20Schwarz=C3=A4ugl?= Date: Tue, 25 Nov 2025 19:47:44 +0100 Subject: [PATCH] feat[server]: add kexec tarball for low-ram bootstrap --- .../nixos/x86_64-linux/milkywell/default.nix | 46 +++++++ .../x86_64-linux/milkywell/disk-config.nix | 121 ++++++++++++++++++ .../milkywell/hardware-configuration.nix | 26 ++++ .../milkywell/secrets/pii.nix.enc | 22 ++++ install/kexec.nix | 96 ++++++++++++++ nix/iso.nix | 37 ++++-- 6 files changed, 336 insertions(+), 12 deletions(-) create mode 100644 hosts/nixos/x86_64-linux/milkywell/default.nix create mode 100644 hosts/nixos/x86_64-linux/milkywell/disk-config.nix create mode 100644 hosts/nixos/x86_64-linux/milkywell/hardware-configuration.nix create mode 100644 hosts/nixos/x86_64-linux/milkywell/secrets/pii.nix.enc create mode 100644 install/kexec.nix diff --git a/hosts/nixos/x86_64-linux/milkywell/default.nix b/hosts/nixos/x86_64-linux/milkywell/default.nix new file mode 100644 index 0000000..3945349 --- /dev/null +++ b/hosts/nixos/x86_64-linux/milkywell/default.nix @@ -0,0 +1,46 @@ +{ lib, config, minimal, ... }: +{ + imports = [ + ./hardware-configuration.nix + ./disk-config.nix + ]; + node.lockFromBootstrapping = false; + sops = { + age.sshKeyPaths = lib.mkDefault [ "/etc/ssh/ssh_host_ed25519_key" ]; + }; + + topology.self = { + icon = "devices.cloud-server"; + }; + + networking = { + domain = "subnet03112148.vcn03112148.oraclevcn.com"; + firewall = { + allowedTCPPorts = [ 53 ]; + }; + }; + + system.stateVersion = "23.11"; + + swarselsystems = { + flakePath = "/root/.dotfiles"; + info = "VM.Standard.E2.1.Micro"; + isImpermanence = true; + isSecureBoot = false; + isCrypted = false; + isSwap = true; + swapSize = "8G"; + rootDisk = "/dev/sda"; + isBtrfs = true; + isNixos = true; + isLinux = true; + server = { + inherit (config.repo.secrets.local.networking) localNetwork; + }; + }; +} // lib.optionalAttrs (!minimal) { + swarselprofiles = { + server = true; + }; + +} diff --git a/hosts/nixos/x86_64-linux/milkywell/disk-config.nix b/hosts/nixos/x86_64-linux/milkywell/disk-config.nix new file mode 100644 index 0000000..9a98cce --- /dev/null +++ b/hosts/nixos/x86_64-linux/milkywell/disk-config.nix @@ -0,0 +1,121 @@ +{ lib, pkgs, config, ... }: +let + type = "btrfs"; + extraArgs = [ "-L" "nixos" "-f" ]; # force overwrite + subvolumes = { + "/root" = { + mountpoint = "/"; + mountOptions = [ + "subvol=root" + "compress=zstd" + "noatime" + ]; + }; + "/home" = lib.mkIf config.swarselsystems.isImpermanence { + mountpoint = "/home"; + mountOptions = [ + "subvol=home" + "compress=zstd" + "noatime" + ]; + }; + "/persist" = lib.mkIf config.swarselsystems.isImpermanence { + mountpoint = "/persist"; + mountOptions = [ + "subvol=persist" + "compress=zstd" + "noatime" + ]; + }; + "/log" = lib.mkIf config.swarselsystems.isImpermanence { + mountpoint = "/var/log"; + mountOptions = [ + "subvol=log" + "compress=zstd" + "noatime" + ]; + }; + "/nix" = { + mountpoint = "/nix"; + mountOptions = [ + "subvol=nix" + "compress=zstd" + "noatime" + ]; + }; + "/swap" = lib.mkIf config.swarselsystems.isSwap { + mountpoint = "/.swapvol"; + swap.swapfile.size = config.swarselsystems.swapSize; + }; + }; +in +{ + disko = { + imageBuilder.extraDependencies = [ pkgs.kmod ]; + devices = { + disk = { + disk0 = { + type = "disk"; + device = config.swarselsystems.rootDisk; + content = { + type = "gpt"; + partitions = { + ESP = { + priority = 1; + name = "ESP"; + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "defaults" ]; + }; + }; + root = lib.mkIf (!config.swarselsystems.isCrypted) { + size = "100%"; + content = { + inherit type subvolumes extraArgs; + postCreateHook = lib.mkIf config.swarselsystems.isImpermanence '' + MNTPOINT=$(mktemp -d) + mount "/dev/disk/by-label/nixos" "$MNTPOINT" -o subvolid=5 + trap 'umount $MNTPOINT; rm -rf $MNTPOINT' EXIT + btrfs subvolume snapshot -r $MNTPOINT/root $MNTPOINT/root-blank + ''; + }; + }; + luks = lib.mkIf config.swarselsystems.isCrypted { + size = "100%"; + content = { + type = "luks"; + name = "cryptroot"; + passwordFile = "/tmp/disko-password"; # this is populated by bootstrap.sh + settings = { + allowDiscards = true; + # https://github.com/hmajid2301/dotfiles/blob/a0b511c79b11d9b4afe2a5e2b7eedb2af23e288f/systems/x86_64-linux/framework/disks.nix#L36 + crypttabExtraOpts = [ + "fido2-device=auto" + "token-timeout=10" + ]; + }; + content = { + inherit type subvolumes extraArgs; + postCreateHook = lib.mkIf config.swarselsystems.isImpermanence '' + MNTPOINT=$(mktemp -d) + mount "/dev/mapper/cryptroot" "$MNTPOINT" -o subvolid=5 + trap 'umount $MNTPOINT; rm -rf $MNTPOINT' EXIT + btrfs subvolume snapshot -r $MNTPOINT/root $MNTPOINT/root-blank + ''; + }; + }; + }; + }; + }; + }; + }; + }; + }; + + fileSystems."/persist".neededForBoot = lib.mkIf config.swarselsystems.isImpermanence true; + fileSystems."/home".neededForBoot = lib.mkIf config.swarselsystems.isImpermanence true; +} diff --git a/hosts/nixos/x86_64-linux/milkywell/hardware-configuration.nix b/hosts/nixos/x86_64-linux/milkywell/hardware-configuration.nix new file mode 100644 index 0000000..61b6d63 --- /dev/null +++ b/hosts/nixos/x86_64-linux/milkywell/hardware-configuration.nix @@ -0,0 +1,26 @@ +{ lib, modulesPath, ... }: + +{ + imports = + [ + (modulesPath + "/profiles/qemu-guest.nix") + ]; + + boot = { + initrd = { + availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" ]; + kernelModules = [ "dm-snapshot" ]; + }; + kernelModules = [ "kvm-amd" ]; + extraModulePackages = [ ]; + }; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.ens3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; +} diff --git a/hosts/nixos/x86_64-linux/milkywell/secrets/pii.nix.enc b/hosts/nixos/x86_64-linux/milkywell/secrets/pii.nix.enc new file mode 100644 index 0000000..d5c5740 --- /dev/null +++ b/hosts/nixos/x86_64-linux/milkywell/secrets/pii.nix.enc @@ -0,0 +1,22 @@ +{ + "data": "ENC[AES256_GCM,data:pikVONWg81bulGRM+enUyBGxFw4C51177O3WrhRzvCiWpulc9RHVH12AfVh4uAkoOANrPyLZuEUGdu8hvFgcTWBzJXPSPZ2sMfAjx593RhSsW+VM58IS0Oa+N9XxwpYnKiUBHrvAxgJD3vhVgCIrL+1LSylZG/RF4Wj6fw4dkttlCqioRkyNPufBtpMEN+MTw9IFsAKaCV5LTFnetIzm6wUPepExaPOpdFhf61JYyzsHkKZslA4FKs1cWgK5ggJkfQS1aA3KCh8pU8vU6uXRgl4ixyUzn/l1HnGZRhFxjcc0x4RqrJYFW9Qyj7oehHU6AxuzD2bf0vuVxZn7nQy8MrDfsvYL5yU34MSdokwrvg3IBXvPogbmjWoBLl6+0WSlV5s6o8GbTkhFi0kWv5H9AKcywY54ltyzoxAQ/9hsZa9IIGCNMFjVfcrKPQKAfrdLbYQxdioq62lHX0LbXKWU9WPRhiG9eRsETudIPan8VRMvHx/6qS6bXSaEjYkKsVSmUOhcVYp7bp58wl9JKu4qYUOLyE2T00IdmbUWrQ+MP14lj6XKNyLN6//8qFyTyhDBLPBJ5DxOMt6qn1qe/lqD4R8Iqvvj63IFq3/psdDXHu+WNmq2/LQH3Y8GTxIoEQ+uPK/I7Tqdh2DXQgKYK162ybczcqsQwGwQWe+DWtno8fauypNrFp6Wgd7c62pxY+8nJzTU9gYSyAIHWEHmRW1LV231X+7kB+JwB2AxfDFXRChuNtJkOK9pdwo4SSS/tGDS5RLmsO4VpE2mPmhxA78IuU88c0LF/e4fC8N5hteecXjMMsSswbC1VUO/3B0JfN8nI0/BnE5Hhau8McSZ1Z06/GsxC+6ArX+zIXthcLIr95nAnLDgJUOKOU7XvLQ9Y22u4lmYWedSTg+vjdljjUG3aBNyp5ZI2xXQmOOgL21aE9aQgBN27GQGqgzECrNEbK0osBefjw14sxv/aZlWU9hHlVROLVbpyWl3edN9ZEMYSJDCpMPwFkEOshlkZ78IfQw7fWRKoBW3W9uHyRPEFeldy8KQ2Lux8A5KwokgFFrp4JI92gjUSubfFsCZp/NHrV5k5UltfMz50QdaKw==,iv:5tRqYZwfz4AeC/HSetPfDaysniUoAgklLl7mEiWBqiM=,tag:7TnVeBMtP8Q81eqeRu02gg==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1glge4e97vgqzh332mqs5990vteezu2m8k4wq3z35jk0q8czw3gks2d7a3h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJZzY0QVQ4ZUxxZkdhQ2Zn\nOHpmTnRaR0R3cXh2Z2JFM1RDVDB2QnE3M3prCm43NjQyOS93UTZKaUlUUmhVcTdG\nUWp1YU1kVmZPc0tBN2FMY2FFVkI1a0UKLS0tIFovZi9FQlhMaXpvcnRYN2FiSm16\nTzJESjNyZ1NzajJRNDR6ZTd2TitoQTgKe2hC6OpYIzgqzhmeJuHWe0yXNE+/Ek26\nGt7s1B6OKnrj+S3es84ePOjAbLHr/ez282b/h0y55ws4R7jMemUIrQ==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-11-25T15:37:26Z", + "mac": "ENC[AES256_GCM,data:nZoyO4iZKAgecFiQ0tBdTEogMIDhe+Tg28L73DLVGCDTaG4QTR4ulvh77R3+Guun6eV5CsH86hTgENgDvybEVJV9bZmJWVbVQ0a+QYsZKIVDcH+o1ZK5EiOoaUb+Tfze1CGey2aw8zBgz3hl7ZeVjb5XNsKOhQz00Oc8xQ7z2CQ=,iv:x7oSgep++DVe2JQ1PPORcqfgBpCRbjO+MNPpDVSlzeI=,tag:JlibH3nTCf64bqxpnCxJAg==,type:str]", + "pgp": [ + { + "created_at": "2025-11-20T01:03:05Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhQIMAwDh3VI7VctTAQ//eTxMD8ZbwJUqVsi1IKK2qdprLTjE0rqdDue+OvP0V+Ns\n1uTnw+b2UBykbIofXcG4P61OxAFdEs8whiIdffQtkDTkOgzV9IQCBOSGxZGEJXMe\nrl5BZLlF98JZ5R15v8V8vMwWwtC90GZ7gZLDV+yZz40Zqm3mTrFz/3PERukwu4Gb\nLTJDOsGmpooyI8KnrIsBhfEwo7/ouAayuKQfvt2i2Tngk9Em73R91BlpcxsOEmqr\n5KWA4GCsjUOmZZKLj2vyENPgQh8t8bP5fGJ3Rf4J1MCWAB89omcE0aRWId/l5sdA\n/Nxinh3xQsiXHPzPLZQ+UjHs+MjNdUoZapoDBP84j2tHsSxh0RMRhlHpESDWq3Mm\n1acWrChyyds6Lz5ZqkioqvAZ3lslS0kPdQqfsLzYWBhA9kLOIJKYfat+vxsAPwAa\n6kceXtxSzUpThtDUPDibjomn7Mrj7ZoHhiJZup/M27glf/V4P3zk+ctpXMSIE7Ia\nQ/jgRDzpcs+u05RsP32jFbCAfi//WxRo77MoxGMJxDhYibBp+aRkFAgVYiElhxbt\n/NedcIAHSJZFyDPm0wn411+DPnUTPn9D9LCkmSG68ZeGDGZJl7Sz3bJ3obWWecTG\nBjqxMZVwRuU2gdg1IwempP9u1dP0Q+g8B3veui/gczGx3J5kvNv8hnUBTeUl2EyF\nAgwDC9FRLmchgYQBD/oCciOvXMrH9/hWIIYb1sKiuCmgdVfs7H0q92XdVNgkbPRz\nXAakX7dl5cZt748u/eCHlGUGr4q7yA1tDx9Vm/J+O2HljN3lBVCbm7HP+YcI+5g0\nvvxr0cIPtr5CXlZz6hJjTgzE4HfEKagGdjgllbHYBB+0rtq/2pZTa20fG0w4coeI\nB/D0iVFwyuM3Wxt/7gXpPtI+m/3qt8QoFIGsZkck7X5hdJwGF4DD5jKxYB28s5Hc\n4ZBG19jezjMIVJUGE58TTVDTvZvJ5Vaw2RizV8DRkFS3q0UIOapOESpZiRnoOqA1\nDQpAU26RSEj8wlYsgNrVWUpdwlYs5e3EWYNkGROTRSB/dGcCSVF31A76W7af+6uv\nwZdMCrAGlD4GBj/yojdnqstfB2Jxu99VubcImWKfaJEXYx5xoREGmK9+t896GJi+\nE8mjiMOMRZFV2n2nwTxAFMaiDJ+VpKpKGVKCOSDwqsePhY/A4kb+N1nnhutmSl/v\n1SCDDvC9+jYNLUC1IaJfFOrNClA43IdJELOAavRx2t1RdyfyOx3D8rrWhF4+NB9Z\nlAc2e7hOoP/OEtf4YjZWq3dQtWSdwePWBxD9xyvF/kEmd2NcezqdfggH3g84qBxy\nUxBDD3ojMMAXlkPU3hRiDeLd1mHxDizVxqYkIYDSeAKtuv2ECH8y7/mv3sKrFtJe\nAQvSMW7gOmIdtQaIpsXHMxzXf+Nv0l3dZeWYD/TnVvoeVOaRQ9dHrtl3J0U9UN3j\nBOJdFaptlS4SIRkva6v6srrM7dXKvjR6IabdzaWl098VW9RFD+YGJ6ZhuQ+zOA==\n=l0k2\n-----END PGP MESSAGE-----", + "fp": "4BE7925262289B476DBBC17B76FD3810215AE097" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.11.0" + } +} diff --git a/install/kexec.nix b/install/kexec.nix new file mode 100644 index 0000000..fc704d8 --- /dev/null +++ b/install/kexec.nix @@ -0,0 +1,96 @@ +{ lib, pkgs, modulesPath, options, ... }: +{ + disabledModules = [ + # This module adds values to multiple lists (systemPackages, supportedFilesystems) + # which are impossible/unpractical to remove, so we disable the entire module. + "profiles/base.nix" + ]; + + imports = [ + # reduce closure size by removing perl + "${modulesPath}/profiles/perlless.nix" + # FIXME: we still are left with nixos-generate-config due to nixos-install-tools + { system.forbiddenDependenciesRegexes = lib.mkForce [ ]; } + ]; + + config = { + networking.hostName = "brickroad"; + + system = { + # nixos-option is mainly useful for interactive installations + tools.nixos-option.enable = false; + # among others, this prevents carrying a stdenv with gcc in the image + extraDependencies = lib.mkForce [ ]; + }; + # prevents shipping nixpkgs, unnecessary if system is evaluated externally + nix.registry = lib.mkForce { }; + + # would pull in nano + programs.nano.enable = false; + + # prevents strace + environment = { + defaultPackages = lib.mkForce [ + pkgs.parted + pkgs.gptfdisk + pkgs.e2fsprogs + ]; + + systemPackages = with pkgs; [ + cryptsetup.bin + ]; + + # Don't install the /lib/ld-linux.so.2 stub. This saves one instance of nixpkgs. + ldso32 = null; + }; + + # included in systemd anyway + systemd.sysusers.enable = true; + + # normal users are not allowed with sys-users + # see https://github.com/NixOS/nixpkgs/pull/328926 + users.users.nixos = { + isSystemUser = true; + isNormalUser = lib.mkForce false; + shell = "/run/current-system/sw/bin/bash"; + group = "nixos"; + }; + users.groups.nixos = { }; + + security = { + # we have still run0 from systemd and most of the time we just use root + sudo.enable = false; + polkit.enable = lib.mkForce false; + # introduces x11 dependencies + pam.services.su.forwardXAuth = lib.mkForce false; + }; + + documentation = { + enable = false; + man.enable = false; + nixos.enable = false; + info.enable = false; + doc.enable = false; + }; + + services = { + # no dependency on x11 + dbus.implementation = "broker"; + # we prefer root as this is also what we use in nixos-anywhere + getty.autologinUser = lib.mkForce "root"; + # included in systemd anyway + userborn.enable = false; + }; + + + + # we are missing this from base.nix + boot.supportedFilesystems = [ + "ext4" + "btrfs" + "xfs" + ]; + } // lib.optionalAttrs (options.hardware ? firmwareCompression) { + hardware.firmwareCompression = "xz"; + }; +} diff --git a/nix/iso.nix b/nix/iso.nix index 75295ad..d2c993c 100644 --- a/nix/iso.nix +++ b/nix/iso.nix @@ -2,19 +2,32 @@ { perSystem = { pkgs, system, ... }: { - # nix build --print-out-paths --no-link .#images..live-iso - packages.live-iso = inputs.nixos-generators.nixosGenerate { - inherit pkgs; - specialArgs = { inherit self; }; - modules = [ - inputs.home-manager.nixosModules.home-manager - "${self}/install/installer-config.nix" - ]; - format = + packages = { + # nix build --print-out-paths --no-link .#live-iso + live-iso = inputs.nixos-generators.nixosGenerate { + inherit pkgs; + specialArgs = { inherit self; }; + modules = [ + inputs.home-manager.nixosModules.home-manager + "${self}/install/installer-config.nix" + ]; + format = + { + x86_64-linux = "install-iso"; + aarch64-linux = "sd-aarch64-installer"; + }.${system}; + }; + + # nix build --print-out-paths --no-link .#pnap-kexec --system + swarsel-kexec = (inputs.smallpkgs.legacyPackages.${system}.nixos [ { - x86_64-linux = "install-iso"; - aarch64-linux = "sd-aarch64-installer"; - }.${system}; + imports = [ "${self}/install/kexec.nix" ]; + _file = __curPos.file; + system.kexec-installer.name = "swarsel-kexec"; + } + inputs.nixos-images.nixosModules.kexec-installer + ]).config.system.build.kexecInstallerTarball; + }; }; }