From eafe3b673c4ff6d1fa83c8a54b78744135ca4d72 Mon Sep 17 00:00:00 2001 From: oddlama Date: Tue, 19 Dec 2023 01:15:51 +0100 Subject: [PATCH] feat(guests): derive stable machine-id for containers; always start sshd in containers --- modules/config/impermanence.nix | 4 +- modules/config/ssh.nix | 3 + modules/guests/common-guest-config.nix | 1 + modules/guests/container.nix | 9 +- modules/guests/default.nix | 99 ++++++++++++------- .../proxy-sentinel/keys/ward-test.pub | 1 - 6 files changed, 74 insertions(+), 43 deletions(-) delete mode 100644 secrets/wireguard/proxy-sentinel/keys/ward-test.pub diff --git a/modules/config/impermanence.nix b/modules/config/impermanence.nix index 78d70e3..a504a40 100644 --- a/modules/config/impermanence.nix +++ b/modules/config/impermanence.nix @@ -109,7 +109,9 @@ in { environment.persistence."/persist" = { hideMounts = true; files = [ - "/etc/machine-id" + # For ephemeral nixos-containers we cannot link the /etc/machine-id file, + # because it will be generated based on a stable container uuid. + (lib.mkIf (!config.boot.isContainer) "/etc/machine-id") "/etc/ssh/ssh_host_ed25519_key" "/etc/ssh/ssh_host_ed25519_key.pub" ]; diff --git a/modules/config/ssh.nix b/modules/config/ssh.nix index bc9ef36..8d328a0 100644 --- a/modules/config/ssh.nix +++ b/modules/config/ssh.nix @@ -1,6 +1,9 @@ {lib, ...}: { services.openssh = { enable = true; + # In containers, this is true by default, but we don't want that + # because we rely on ssh key generation for agenix + startWhenNeeded = lib.mkForce false; authorizedKeysFiles = lib.mkForce ["/etc/ssh/authorized_keys.d/%u"]; settings = { PasswordAuthentication = false; diff --git a/modules/guests/common-guest-config.nix b/modules/guests/common-guest-config.nix index 82a951b..6bb6e92 100644 --- a/modules/guests/common-guest-config.nix +++ b/modules/guests/common-guest-config.nix @@ -12,6 +12,7 @@ in { systemd.network.networks = { "10-${guestCfg.networking.mainLinkName}" = { + matchConfig.Name = guestCfg.networking.mainLinkName; DHCP = "yes"; dhcpV4Config.UseDNS = false; dhcpV6Config.UseDNS = false; diff --git a/modules/guests/container.nix b/modules/guests/container.nix index c312600..40c3f57 100644 --- a/modules/guests/container.nix +++ b/modules/guests/container.nix @@ -17,10 +17,13 @@ guestName: guestCfg: { initialLinkName = "mv-${(substring 0 12 (builtins.hashString "sha256" guestName))}"; in { - autoStart = guestCfg.autostart; - macvlans = ["${guestCfg.container.macvlan}:${initialLinkName}"]; ephemeral = true; privateNetwork = true; + autoStart = guestCfg.autostart; + macvlans = ["${guestCfg.container.macvlan}:${initialLinkName}"]; + extraFlags = [ + "--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}" + ]; bindMounts = flip mapAttrs' guestCfg.zfs ( _: zfsCfg: nameValuePair zfsCfg.guestMountpoint { @@ -64,7 +67,7 @@ in { # Rename the network interface to our liking systemd.network.links = { "01-${guestCfg.networking.mainLinkName}" = { - matchConfig.OriginalName = initialLinkName; + matchConfig.Name = initialLinkName; linkConfig.Name = guestCfg.networking.mainLinkName; }; }; diff --git a/modules/guests/default.nix b/modules/guests/default.nix index aaa7be6..554db85 100644 --- a/modules/guests/default.nix +++ b/modules/guests/default.nix @@ -9,23 +9,30 @@ inherit (lib) attrValues - any + attrsToList disko escapeShellArg - makeBinPath - mapAttrsToList - mkMerge - mergeToplevelConfigs flip + groupBy + listToAttrs + makeBinPath + mapAttrs + mapAttrsToList + mergeToplevelConfigs mkIf + mkMerge mkOption types ; + backends = ["microvm" "container"]; nodeName = config.node.name; + guestsByBackend = + lib.genAttrs backends (_: {}) + // mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests)); # Configuration required on the host for a specific guest - defineGuest = guestName: guestCfg: { + defineGuest = _guestName: guestCfg: { # Add the required datasets to the disko configuration of the machine disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: { ${zfsCfg.pool}.datasets.${zfsCfg.dataset} = @@ -56,33 +63,43 @@ fi ''; }; - - # Ensure that the zfs dataset has the correct permissions when mounted - "zfs-chown-${utils.escapeSystemdPath zfsCfg.hostMountpoint}" = { - after = [fsMountUnit]; - unitConfig.DefaultDependencies = "no"; - serviceConfig.Type = "oneshot"; - script = '' - chmod 700 ${escapeShellArg zfsCfg.hostMountpoint} - ''; - }; - - "microvm@${guestName}" = mkIf (guestCfg.backend == "microvm") { - requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath zfsCfg.hostMountpoint}.service"]; - after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath zfsCfg.hostMountpoint}.service"]; - }; - - "container@${guestName}" = mkIf (guestCfg.backend == "container") { - requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath zfsCfg.hostMountpoint}.service"]; - after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath zfsCfg.hostMountpoint}.service"]; - }; })); + }; - microvm.vms.${guestName} = - mkIf (guestCfg.backend == "microvm") (import ./microvm.nix guestName guestCfg attrs); + defineMicrovm = guestName: guestCfg: { + # Ensure that the zfs dataset exists before it is mounted. + systemd.services."microvm@${guestName}" = let + fsMountUnits = + map + (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") + (attrValues guestCfg.zfs); + in { + requires = fsMountUnits; + after = fsMountUnits; + }; - containers.${guestName} = - mkIf (guestCfg.backend == "container") (import ./container.nix guestName guestCfg attrs); + microvm.vms.${guestName} = import ./microvm.nix guestName guestCfg attrs; + }; + + defineContainer = guestName: guestCfg: { + # Ensure that the zfs dataset exists before it is mounted. + systemd.services."container@${guestName}" = let + fsMountUnits = + map + (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") + (attrValues guestCfg.zfs); + in { + requires = fsMountUnits; + after = fsMountUnits; + # Don't use the notify service type. Using exec will always consider containers + # started immediately and donesn't wait until the container is fully booted. + # Containers should behave like independent machines, and issues inside the container + # will unnecessarily lock up the service on the host otherwise. + # This causes issues on system activation. + serviceConfig.Type = lib.mkForce "exec"; + }; + + containers.${guestName} = import ./container.nix guestName guestCfg attrs; }; in { imports = [ @@ -90,10 +107,7 @@ in { inputs.microvm.nixosModules.host # This is opt-out, so we can't put this into the mkIf below { - microvm.host.enable = - any - (guestCfg: guestCfg.backend == "microvm") - (attrValues config.guests); + microvm.host.enable = guestsByBackend.microvm != {}; } ]; @@ -132,7 +146,7 @@ in { }; backend = mkOption { - type = types.enum ["microvm" "container"]; + type = types.enum backends; description = '' Determines how the guest will be hosted. You can currently choose between microvm based deployment, or nixos containers. @@ -216,7 +230,16 @@ in { })); }; - config = - mkIf (config.guests != {}) - (mergeToplevelConfigs ["containers" "disko" "microvm" "systemd"] (mapAttrsToList defineGuest config.guests)); + config = mkIf (config.guests != {}) ( + mkMerge [ + { + systemd.tmpfiles.rules = [ + "d /guests 0700 root root -" + ]; + } + (mergeToplevelConfigs ["disko" "systemd"] (mapAttrsToList defineGuest config.guests)) + (mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container)) + (mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm)) + ] + ); } diff --git a/secrets/wireguard/proxy-sentinel/keys/ward-test.pub b/secrets/wireguard/proxy-sentinel/keys/ward-test.pub deleted file mode 100644 index 329b759..0000000 --- a/secrets/wireguard/proxy-sentinel/keys/ward-test.pub +++ /dev/null @@ -1 +0,0 @@ -PTlU+qtfddz0ZfcHcfZmSxZ4Abe8UCpWV2FBJQswzBk=