feat: add automatic zfs persistent dataset management to microvms

hosts/common/core/impermanence.nix

@@ -21,8 +21,9 @@
       "/etc/ssh/ssh_host_ed25519_key.pub"
     ];
     directories = [
-      "/var/log"
       "/var/lib/nixos"
+      "/var/lib/systemd/coredump"
+      "/var/log"
     ];
   };
 }

hosts/ward/default.nix

@@ -30,6 +30,12 @@
     macOffset = config.lib.net.mac.addPrivate nodeSecrets.networking.interfaces.lan.mac;
   in {
     test = {
+      zfs = {
+        enable = true;
+        pool = "rpool";
+        dataset = "safe/vms/test";
+        mountpoint = "/persist/vms/test";
+      };
       autostart = true;
       mac = macOffset "00:00:00:00:00:11";
       macvtap = "lan";

hosts/ward/fs.nix

@@ -35,6 +35,7 @@
             "local/nix" = filesystem "/nix";
             "safe" = unmountable;
             "safe/persist" = filesystem "/persist";
+            "safe/vms" = unmountable;
           };
         };
     };
@@ -42,10 +43,6 @@
 
   fileSystems."/persist".neededForBoot = true;
 
-  #environment.persistence."/persist".directories = [
-  #  { directory = "/var/lib/acme"; user = "acme"; group = "acme"; }
-  #];
-
   # After importing the rpool, rollback the root system to be empty.
   boot.initrd.systemd.services = {
     impermanence-root = {

modules/microvms.nix

@@ -13,100 +13,174 @@
     (lib)
     attrNames
     concatStringsSep
+    escapeShellArg
     filterAttrs
-    mapAttrs
+    foldl'
     mapAttrsToList
     mdDoc
     mkDefault
+    mkEnableOption
     mkForce
     mkIf
+    mkMerge
     mkOption
+    optionalAttrs
+    recursiveUpdate
     types
     ;
 
   cfg = config.extra.microvms;
 
-  defineMicrovm = vmName: vmCfg: let
+  # Base configuration required for the host
-    node =
+  hostConfig = {
-      (import ../nix/generate-node.nix inputs)
+    assertions = let
-      "${nodeName}-microvm-${vmName}" {
+      duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
-        inherit (vmCfg) system;
+    in [
-        config = nodePath + "/microvms/${vmName}";
+      {
-      };
+        assertion = duplicateMacs == [];
-  in {
+        message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
-    inherit (node) pkgs specialArgs;
+      }
-    config = {
+    ];
-      imports = [microvm.microvm] ++ node.imports;
 
-      microvm = {
+    microvm = {
-        hypervisor = mkDefault "cloud-hypervisor";
+      declarativeUpdates = true;
+      restartIfChanged = true;
+    };
+  };
 
-        # MACVTAP bridge to the host's network
+  # Configuration for each microvm
-        interfaces = [
+  microvmConfig = vmName: vmCfg: {
-          {
+    # Add the required datasets to the disko configuration of the machine
-            type = "macvtap";
+    disko.devices.zpool = mkIf (vmCfg.zfs.enable && vmCfg.zfs.disko) {
-            id = "vm-${vmName}";
+      ${vmCfg.zfs.pool}.datasets."${vmCfg.zfs.dataset}" =
-            macvtap = {
+        extraLib.disko.zfs.filesystem "${vmCfg.zfs.mountpoint}";
-              link = vmCfg.macvtap;
+    };
-              mode = "bridge";
-            };
-            inherit (vmCfg) mac;
-          }
-        ];
 
-        shares = [
+    # When installing a microvm, make sure that its persitent zfs dataset exists
-          # Share the nix-store of the host
+    systemd.services."install-microvm-${vmName}".preStart = let
-          {
+      poolDataset = "${vmCfg.zfs.pool}/${vmCfg.zfs.dataset}";
-            source = "/nix/store";
+    in
-            mountPoint = "/nix/.ro-store";
+      mkIf vmCfg.zfs.enable ''
-            tag = "ro-store";
+        if ! ${pkgs.zfs}/bin/zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
-            proto = "virtiofs";
+          ${pkgs.zfs}/bin/zfs create -o canmount=on -o mountpoint=${escapeShellArg vmCfg.zfs.mountpoint} ${escapeShellArg poolDataset}
-          }
+        fi
-          # Mount persistent data from the host
+      '';
-          #{
-          #  source = "/persist/vms/${vmName}";
-          #  mountPoint = "/persist";
-          #  tag = "persist";
-          #  proto = "virtiofs";
-          #}
-        ];
-      };
 
-      # Add a writable store overlay, but since this is always ephemeral
+    microvm.autostart = mkIf vmCfg.autostart [vmName];
-      # disable any store optimization from nix.
+    microvm.vms.${vmName} = let
-      microvm.writableStoreOverlay = "/nix/.rw-store";
+      node =
-      nix = {
+        (import ../nix/generate-node.nix inputs)
-        settings.auto-optimise-store = mkForce false;
+        "${nodeName}-microvm-${vmName}" {
-        optimise.automatic = mkForce false;
+          inherit (vmCfg) system;
-        gc.automatic = mkForce false;
+          config = nodePath + "/microvms/${vmName}";
-      };
-
-      extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac;
-
-      systemd.network.networks = {
-        "10-${vmCfg.linkName}" = {
-          matchConfig.Name = vmCfg.linkName;
-          DHCP = "yes";
-          networkConfig = {
-            IPv6PrivacyExtensions = "yes";
-            IPv6AcceptRA = true;
-          };
-          linkConfig.RequiredForOnline = "routable";
         };
-      };
+    in {
+      inherit (node) pkgs specialArgs;
+      config = {
+        imports = [microvm.microvm] ++ node.imports;
 
-      # TODO change once microvms are compatible with stage-1 systemd
+        microvm = {
-      boot.initrd.systemd.enable = mkForce false;
+          hypervisor = mkDefault "cloud-hypervisor";
+
+          # MACVTAP bridge to the host's network
+          interfaces = [
+            {
+              type = "macvtap";
+              id = "vm-${vmName}";
+              macvtap = {
+                link = vmCfg.macvtap;
+                mode = "bridge";
+              };
+              inherit (vmCfg) mac;
+            }
+          ];
+
+          shares =
+            [
+              # Share the nix-store of the host
+              {
+                source = "/nix/store";
+                mountPoint = "/nix/.ro-store";
+                tag = "ro-store";
+                proto = "virtiofs";
+              }
+            ]
+            # Mount persistent data from the host
+            ++ optionalAttrs vmCfg.zfs.enable {
+              source = vmCfg.zfs.mountpoint;
+              mountPoint = "/persist";
+              tag = "persist";
+              proto = "virtiofs";
+            };
+        };
+
+        fileSystems."/persist".neededForBoot = true;
+
+        # Add a writable store overlay, but since this is always ephemeral
+        # disable any store optimization from nix.
+        microvm.writableStoreOverlay = "/nix/.rw-store";
+        nix = {
+          settings.auto-optimise-store = mkForce false;
+          optimise.automatic = mkForce false;
+          gc.automatic = mkForce false;
+        };
+
+        extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac;
+
+        systemd.network.networks = {
+          "10-${vmCfg.linkName}" = {
+            matchConfig.Name = vmCfg.linkName;
+            DHCP = "yes";
+            networkConfig = {
+              IPv6PrivacyExtensions = "yes";
+              IPv6AcceptRA = true;
+            };
+            linkConfig.RequiredForOnline = "routable";
+          };
+        };
+
+        # TODO change once microvms are compatible with stage-1 systemd
+        boot.initrd.systemd.enable = mkForce false;
+      };
     };
   };
 in {
-  imports = [microvm.host];
+  imports = [
+    # Add the host module, but only enable if it necessary
+    microvm.host
+    {microvm.host.enable = cfg != {};}
+  ];
 
   options.extra.microvms = mkOption {
     default = {};
-    description = "Provides a base configuration for MicroVMs.";
+    description = "Handles the necessary base setup for MicroVMs.";
     type = types.attrsOf (types.submodule {
       options = {
+        zfs = {
+          enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset");
+
+          pool = mkOption {
+            type = types.str;
+            description = mdDoc "The host's zfs pool on which the dataset resides";
+          };
+
+          dataset = mkOption {
+            type = types.str;
+            description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
+          };
+
+          mountpoint = mkOption {
+            type = types.str;
+            description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtofs as /persist in the vm)";
+          };
+
+          disko = mkOption {
+            type = types.bool;
+            default = true;
+            description = mdDoc "Add this dataset to the host's disko configuration";
+          };
+        };
+
         autostart = mkOption {
           type = types.bool;
           default = false;
@@ -137,22 +211,8 @@ in {
     });
   };
 
-  config = {
+  config =
-    assertions = let
+    mkIf (cfg != {})
-      duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
+    (extraLib.mkMergeTopLevel ["assertions" "disko" "systemd" "microvm"]
-    in [
+      ([hostConfig] ++ mapAttrsToList microvmConfig cfg));
-      {
-        assertion = duplicateMacs == [];
-        message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
-      }
-    ];
-
-    microvm = {
-      host.enable = cfg != {};
-      declarativeUpdates = true;
-      restartIfChanged = true;
-      vms = mkIf (cfg != {}) (mapAttrs defineMicrovm cfg);
-      autostart = mkIf (cfg != {}) (attrNames (filterAttrs (_: v: v.autostart) cfg));
-    };
-  };
 }

nix/lib.nix

@@ -13,11 +13,15 @@
     escapeShellArg
     filter
     flatten
+    foldAttrs
     foldl'
     genAttrs
+    getAttrs
     head
+    mapAttrs
     mapAttrs'
     mergeAttrs
+    mkMerge
     nameValuePair
     optionalAttrs
     partition
@@ -49,6 +53,13 @@ in rec {
   # True if the path or string starts with /
   isAbsolutePath = x: substring 0 1 x == "/";
 
+  # Used to merge multiple toplevel configuration entries
+  # https://gist.github.com/udf/4d9301bdc02ab38439fd64fbda06ea43
+  mkMergeTopLevel = names: attrs:
+    getAttrs names (
+      mapAttrs (_: mkMerge) (foldAttrs (n: a: [n] ++ a) [] attrs)
+    );
+
   disko = {
     gpt = {
       partEfi = name: start: end: {