diff --git a/hosts/common/core/impermanence.nix b/hosts/common/core/impermanence.nix index 7cdc0f2..07170fb 100644 --- a/hosts/common/core/impermanence.nix +++ b/hosts/common/core/impermanence.nix @@ -21,8 +21,9 @@ "/etc/ssh/ssh_host_ed25519_key.pub" ]; directories = [ - "/var/log" "/var/lib/nixos" + "/var/lib/systemd/coredump" + "/var/log" ]; }; } diff --git a/hosts/ward/default.nix b/hosts/ward/default.nix index c336d09..4cbec23 100644 --- a/hosts/ward/default.nix +++ b/hosts/ward/default.nix @@ -30,6 +30,12 @@ macOffset = config.lib.net.mac.addPrivate nodeSecrets.networking.interfaces.lan.mac; in { test = { + zfs = { + enable = true; + pool = "rpool"; + dataset = "safe/vms/test"; + mountpoint = "/persist/vms/test"; + }; autostart = true; mac = macOffset "00:00:00:00:00:11"; macvtap = "lan"; diff --git a/hosts/ward/fs.nix b/hosts/ward/fs.nix index c760c6d..0839f4a 100644 --- a/hosts/ward/fs.nix +++ b/hosts/ward/fs.nix @@ -35,6 +35,7 @@ "local/nix" = filesystem "/nix"; "safe" = unmountable; "safe/persist" = filesystem "/persist"; + "safe/vms" = unmountable; }; }; }; @@ -42,10 +43,6 @@ fileSystems."/persist".neededForBoot = true; - #environment.persistence."/persist".directories = [ - # { directory = "/var/lib/acme"; user = "acme"; group = "acme"; } - #]; - # After importing the rpool, rollback the root system to be empty. boot.initrd.systemd.services = { impermanence-root = { diff --git a/modules/microvms.nix b/modules/microvms.nix index 158f89a..f76dea9 100644 --- a/modules/microvms.nix +++ b/modules/microvms.nix @@ -13,100 +13,174 @@ (lib) attrNames concatStringsSep + escapeShellArg filterAttrs - mapAttrs + foldl' mapAttrsToList mdDoc mkDefault + mkEnableOption mkForce mkIf + mkMerge mkOption + optionalAttrs + recursiveUpdate types ; cfg = config.extra.microvms; - defineMicrovm = vmName: vmCfg: let - node = - (import ../nix/generate-node.nix inputs) - "${nodeName}-microvm-${vmName}" { - inherit (vmCfg) system; - config = nodePath + "/microvms/${vmName}"; - }; - in { - inherit (node) pkgs specialArgs; - config = { - imports = [microvm.microvm] ++ node.imports; + # Base configuration required for the host + hostConfig = { + assertions = let + duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg); + in [ + { + assertion = duplicateMacs == []; + message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}"; + } + ]; - microvm = { - hypervisor = mkDefault "cloud-hypervisor"; + microvm = { + declarativeUpdates = true; + restartIfChanged = true; + }; + }; - # MACVTAP bridge to the host's network - interfaces = [ - { - type = "macvtap"; - id = "vm-${vmName}"; - macvtap = { - link = vmCfg.macvtap; - mode = "bridge"; - }; - inherit (vmCfg) mac; - } - ]; + # Configuration for each microvm + microvmConfig = vmName: vmCfg: { + # Add the required datasets to the disko configuration of the machine + disko.devices.zpool = mkIf (vmCfg.zfs.enable && vmCfg.zfs.disko) { + ${vmCfg.zfs.pool}.datasets."${vmCfg.zfs.dataset}" = + extraLib.disko.zfs.filesystem "${vmCfg.zfs.mountpoint}"; + }; - shares = [ - # Share the nix-store of the host - { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "ro-store"; - proto = "virtiofs"; - } - # Mount persistent data from the host - #{ - # source = "/persist/vms/${vmName}"; - # mountPoint = "/persist"; - # tag = "persist"; - # proto = "virtiofs"; - #} - ]; - }; + # When installing a microvm, make sure that its persitent zfs dataset exists + systemd.services."install-microvm-${vmName}".preStart = let + poolDataset = "${vmCfg.zfs.pool}/${vmCfg.zfs.dataset}"; + in + mkIf vmCfg.zfs.enable '' + if ! ${pkgs.zfs}/bin/zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then + ${pkgs.zfs}/bin/zfs create -o canmount=on -o mountpoint=${escapeShellArg vmCfg.zfs.mountpoint} ${escapeShellArg poolDataset} + fi + ''; - # Add a writable store overlay, but since this is always ephemeral - # disable any store optimization from nix. - microvm.writableStoreOverlay = "/nix/.rw-store"; - nix = { - settings.auto-optimise-store = mkForce false; - optimise.automatic = mkForce false; - gc.automatic = mkForce false; - }; - - extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac; - - systemd.network.networks = { - "10-${vmCfg.linkName}" = { - matchConfig.Name = vmCfg.linkName; - DHCP = "yes"; - networkConfig = { - IPv6PrivacyExtensions = "yes"; - IPv6AcceptRA = true; - }; - linkConfig.RequiredForOnline = "routable"; + microvm.autostart = mkIf vmCfg.autostart [vmName]; + microvm.vms.${vmName} = let + node = + (import ../nix/generate-node.nix inputs) + "${nodeName}-microvm-${vmName}" { + inherit (vmCfg) system; + config = nodePath + "/microvms/${vmName}"; }; - }; + in { + inherit (node) pkgs specialArgs; + config = { + imports = [microvm.microvm] ++ node.imports; - # TODO change once microvms are compatible with stage-1 systemd - boot.initrd.systemd.enable = mkForce false; + microvm = { + hypervisor = mkDefault "cloud-hypervisor"; + + # MACVTAP bridge to the host's network + interfaces = [ + { + type = "macvtap"; + id = "vm-${vmName}"; + macvtap = { + link = vmCfg.macvtap; + mode = "bridge"; + }; + inherit (vmCfg) mac; + } + ]; + + shares = + [ + # Share the nix-store of the host + { + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + } + ] + # Mount persistent data from the host + ++ optionalAttrs vmCfg.zfs.enable { + source = vmCfg.zfs.mountpoint; + mountPoint = "/persist"; + tag = "persist"; + proto = "virtiofs"; + }; + }; + + fileSystems."/persist".neededForBoot = true; + + # Add a writable store overlay, but since this is always ephemeral + # disable any store optimization from nix. + microvm.writableStoreOverlay = "/nix/.rw-store"; + nix = { + settings.auto-optimise-store = mkForce false; + optimise.automatic = mkForce false; + gc.automatic = mkForce false; + }; + + extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac; + + systemd.network.networks = { + "10-${vmCfg.linkName}" = { + matchConfig.Name = vmCfg.linkName; + DHCP = "yes"; + networkConfig = { + IPv6PrivacyExtensions = "yes"; + IPv6AcceptRA = true; + }; + linkConfig.RequiredForOnline = "routable"; + }; + }; + + # TODO change once microvms are compatible with stage-1 systemd + boot.initrd.systemd.enable = mkForce false; + }; }; }; in { - imports = [microvm.host]; + imports = [ + # Add the host module, but only enable if it necessary + microvm.host + {microvm.host.enable = cfg != {};} + ]; options.extra.microvms = mkOption { default = {}; - description = "Provides a base configuration for MicroVMs."; + description = "Handles the necessary base setup for MicroVMs."; type = types.attrsOf (types.submodule { options = { + zfs = { + enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset"); + + pool = mkOption { + type = types.str; + description = mdDoc "The host's zfs pool on which the dataset resides"; + }; + + dataset = mkOption { + type = types.str; + description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)"; + }; + + mountpoint = mkOption { + type = types.str; + description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtofs as /persist in the vm)"; + }; + + disko = mkOption { + type = types.bool; + default = true; + description = mdDoc "Add this dataset to the host's disko configuration"; + }; + }; + autostart = mkOption { type = types.bool; default = false; @@ -137,22 +211,8 @@ in { }); }; - config = { - assertions = let - duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg); - in [ - { - assertion = duplicateMacs == []; - message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}"; - } - ]; - - microvm = { - host.enable = cfg != {}; - declarativeUpdates = true; - restartIfChanged = true; - vms = mkIf (cfg != {}) (mapAttrs defineMicrovm cfg); - autostart = mkIf (cfg != {}) (attrNames (filterAttrs (_: v: v.autostart) cfg)); - }; - }; + config = + mkIf (cfg != {}) + (extraLib.mkMergeTopLevel ["assertions" "disko" "systemd" "microvm"] + ([hostConfig] ++ mapAttrsToList microvmConfig cfg)); } diff --git a/nix/lib.nix b/nix/lib.nix index 0987c94..64e0ca6 100644 --- a/nix/lib.nix +++ b/nix/lib.nix @@ -13,11 +13,15 @@ escapeShellArg filter flatten + foldAttrs foldl' genAttrs + getAttrs head + mapAttrs mapAttrs' mergeAttrs + mkMerge nameValuePair optionalAttrs partition @@ -49,6 +53,13 @@ in rec { # True if the path or string starts with / isAbsolutePath = x: substring 0 1 x == "/"; + # Used to merge multiple toplevel configuration entries + # https://gist.github.com/udf/4d9301bdc02ab38439fd64fbda06ea43 + mkMergeTopLevel = names: attrs: + getAttrs names ( + mapAttrs (_: mkMerge) (foldAttrs (n: a: [n] ++ a) [] attrs) + ); + disko = { gpt = { partEfi = name: start: end: {