feat: wip: begin modularizing microvm definitions to allow for other

guest types like containers

hosts/kroma/default.nix

@@ -42,6 +42,9 @@
   '';
   networking.extraHosts = "127.0.0.1 modules-cdn.eac-prod.on.epicgames.com";
 
+  systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
+  systemd.services."systemd-resolved".environment.SYSTEMD_LOG_LEVEL = "debug";
+
   graphical.gaming.enable = true;
 
   stylix.fonts.sizes = {
@@ -50,4 +53,7 @@
     terminal = 20;
     popups = 20;
   };
+
+  nix.settings.trusted-substituters = ["https://ai.cachix.org"];
+  nix.settings.trusted-public-keys = ["ai.cachix.org-1:N9dzRK+alWwoKXQlnn0H6aUx0lU/mspIoz8hMvGvbbc="];
 }

hosts/ward/default.nix

@@ -48,6 +48,39 @@
     ./microvms/common.nix
   ];
 
+  #guests.adguardhome = {
+  #  backend = "microvm";
+  #  microvm = {
+  #    system = "x86_64-linux";
+  #    autostart = true;
+  #  };
+  #  zfs = {
+  #    enable = true;
+  #    pool = "rpool";
+  #  };
+  #  modules = [ ./guests/adguardhome.nix ];
+  #};
+
+  guests = let
+    mkMicrovm = system: module: {
+      backend = "microvm";
+      microvm = {
+        system = "x86_64-linux";
+        autostart = true;
+      };
+      zfs = {
+        enable = true;
+        pool = "rpool";
+      };
+      modules = [
+        ../../modules
+        module
+      ];
+    };
+  in {
+    adguardhome = mkMicrovm "x86_64-linux" ./guests/adguardhome.nix;
+  };
+
   meta.microvms.vms = let
     defaultConfig = name: {
       system = "x86_64-linux";

hosts/ward/net.nix

@@ -127,7 +127,6 @@ in {
   meta.microvms.networking = {
     baseMac = config.repo.secrets.local.networking.interfaces.lan.mac;
     macvtapInterface = "lan";
-    wireguard.openFirewallRules = ["lan-to-local"];
   };
 
   # Allow accessing influx

modules/config/issue.nix

@@ -1,10 +1,14 @@
-{lib, ...} @ args: {
+{
+  config,
+  lib,
+  ...
+}: {
   # IP addresses: ${"${interface} \e{halfbright}\4{${interface}}\e{reset} \e{halfbright}\6{${interface}}\e{reset}"}
   environment.etc.issue.text = lib.concatStringsSep "\n" ([
       ''\d  \t''
       ''This is \e{cyan}\n\e{reset} [\e{lightblue}\l\e{reset}] (\s \m \r)''
     ]
-    # Disabled for microvms because of frequent redraws (-> pushed to syslog on the host)
+    # Disabled for guests because of frequent redraws (-> pushed to syslog on the host)
-    ++ lib.optional (!(args ? parentNodeName)) ''\e{halfbright}\4\e{reset} \e{halfbright}\6\e{reset}''
+    ++ lib.optional (!config.guests.isGuest) ''\e{halfbright}\4\e{reset} \e{halfbright}\6\e{reset}''
     ++ [""]);
 }

modules/meta/microvms.nix

@@ -10,11 +10,12 @@
   inherit
     (lib)
     attrNames
+    attrValues
+    any
     disko
     escapeShellArg
     makeBinPath
     mapAttrsToList
-    mdDoc
     mergeToplevelConfigs
     mkDefault
     mkEnableOption
@@ -26,36 +27,36 @@
     types
     ;
 
-  cfg = config.meta.microvms;
+  cfg = config.guests;
   nodeName = config.node.name;
-  inherit (cfg) vms;
+  inherit (cfg) guests;
 
-  # Configuration for each microvm
+  # Configuration required on the host for a specific guest
-  microvmConfig = vmName: vmCfg: {
+  defineGuest = guestName: guestCfg: {
     # Add the required datasets to the disko configuration of the machine
-    disko.devices.zpool = mkIf vmCfg.zfs.enable {
+    disko.devices.zpool = mkIf guestCfg.zfs.enable {
-      ${vmCfg.zfs.pool}.datasets.${vmCfg.zfs.dataset} =
+      ${guestCfg.zfs.pool}.datasets.${guestCfg.zfs.dataset} =
-        disko.zfs.filesystem vmCfg.zfs.mountpoint;
+        disko.zfs.filesystem guestCfg.zfs.mountpoint;
     };
 
     # Ensure that the zfs dataset exists before it is mounted.
     systemd.services = let
-      fsMountUnit = "${utils.escapeSystemdPath vmCfg.zfs.mountpoint}.mount";
+      fsMountUnit = "${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.mount";
     in
-      mkIf vmCfg.zfs.enable {
+      mkIf guestCfg.zfs.enable {
         # Ensure that the zfs dataset exists before it is mounted.
-        "zfs-ensure-${utils.escapeSystemdPath vmCfg.zfs.mountpoint}" = {
+        "zfs-ensure-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}" = {
           wantedBy = [fsMountUnit];
           before = [fsMountUnit];
           after = [
-            "zfs-import-${utils.escapeSystemdPath vmCfg.zfs.pool}.service"
+            "zfs-import-${utils.escapeSystemdPath guestCfg.zfs.pool}.service"
             "zfs-mount.target"
           ];
           unitConfig.DefaultDependencies = "no";
           serviceConfig.Type = "oneshot";
           script = let
-            poolDataset = "${vmCfg.zfs.pool}/${vmCfg.zfs.dataset}";
+            poolDataset = "${guestCfg.zfs.pool}/${guestCfg.zfs.dataset}";
-            diskoDataset = config.disko.devices.zpool.${vmCfg.zfs.pool}.datasets.${vmCfg.zfs.dataset};
+            diskoDataset = config.disko.devices.zpool.${guestCfg.zfs.pool}.datasets.${guestCfg.zfs.dataset};
           in ''
             export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
             if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
@@ -65,244 +66,237 @@
         };
 
         # Ensure that the zfs dataset has the correct permissions when mounted
-        "zfs-chown-${utils.escapeSystemdPath vmCfg.zfs.mountpoint}" = {
+        "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}" = {
           after = [fsMountUnit];
           unitConfig.DefaultDependencies = "no";
           serviceConfig.Type = "oneshot";
           script = ''
-            chmod 700 ${escapeShellArg vmCfg.zfs.mountpoint}
+            chmod 700 ${escapeShellArg guestCfg.zfs.mountpoint}
           '';
         };
 
-        "microvm@${vmName}" = {
+        "microvm@${guestName}" = mkIf (guestCfg.backend == "microvm") {
-          requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath vmCfg.zfs.mountpoint}.service"];
+          requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
-          after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath vmCfg.zfs.mountpoint}.service"];
+          after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
         };
       };
 
-    microvm.vms.${vmName} = let
+    microvm.vms.${guestName} = let
-      mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames vms)).${vmName};
+      mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames guests)).${guestName};
-    in {
+    in
-      # Allow children microvms to know which node is their parent
+      mkIf (guestCfg.backend == "microvm") {
-      specialArgs = {
+        # Allow children microvms to know which node is their parent
-        parentNode = config;
+        specialArgs = {
-        parentNodeName = nodeName;
+          parentNode = config;
-        inherit (inputs.self) nodes;
+          inherit (inputs.self) nodes;
-        inherit (inputs.self.pkgs.${vmCfg.system}) lib;
+          inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib;
-        inherit inputs;
+          inherit inputs;
-        inherit minimal;
+          inherit minimal;
-      };
+        };
-      pkgs = inputs.self.pkgs.${vmCfg.system};
+        pkgs = inputs.self.pkgs.${guestCfg.microvm.system};
-      inherit (vmCfg) autostart;
+        inherit (guestCfg) autostart;
-      config = {
+        config = {
-        imports = cfg.commonImports ++ vmCfg.modules;
+          imports = guestCfg.modules;
-        node.name = vmCfg.nodeName;
+          node.name = guestCfg.nodeName;
+          node.isGuest = true;
 
-        # TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
+          # TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
-        environment.noXlibs = mkForce false;
+          environment.noXlibs = mkForce false;
-        lib.microvm.mac = mac;
+          lib.microvm.mac = mac;
 
-        microvm = {
+          microvm = {
-          hypervisor = mkDefault "qemu";
+            hypervisor = mkDefault "qemu";
 
-          # Give them some juice by default
+            # Give them some juice by default
-          mem = mkDefault (2 * 1024);
+            mem = mkDefault (2 * 1024);
 
-          # MACVTAP bridge to the host's network
+            # MACVTAP bridge to the host's network
-          interfaces = [
+            interfaces = [
-            {
+              {
-              type = "macvtap";
+                type = "macvtap";
-              id = "vm-${vmName}";
+                id = "vm-${guestName}";
-              inherit mac;
+                inherit mac;
-              macvtap = {
+                macvtap = {
-                link = cfg.networking.macvtapInterface;
+                  link = cfg.networking.macvtapInterface;
-                mode = "bridge";
+                  mode = "bridge";
+                };
+              }
+            ];
+
+            shares =
+              [
+                # Share the nix-store of the host
+                {
+                  source = "/nix/store";
+                  mountPoint = "/nix/.ro-store";
+                  tag = "ro-store";
+                  proto = "virtiofs";
+                }
+                {
+                  source = "/state/guests/${guestName}";
+                  mountPoint = "/state";
+                  tag = "state";
+                  proto = "virtiofs";
+                }
+              ]
+              # Mount persistent data from the host
+              ++ optional guestCfg.zfs.enable {
+                source = guestCfg.zfs.mountpoint;
+                mountPoint = "/persist";
+                tag = "persist";
+                proto = "virtiofs";
               };
-            }
+          };
-          ];
 
-          shares =
+          # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here
-            [
+          fileSystems."/state".neededForBoot = mkForce true;
-              # Share the nix-store of the host
+          fileSystems."/persist".neededForBoot = mkForce true;
-              {
+
-                source = "/nix/store";
+          # Add a writable store overlay, but since this is always ephemeral
-                mountPoint = "/nix/.ro-store";
+          # disable any store optimization from nix.
-                tag = "ro-store";
+          microvm.writableStoreOverlay = "/nix/.rw-store";
-                proto = "virtiofs";
+          nix = {
-              }
+            settings.auto-optimise-store = mkForce false;
-              {
+            optimise.automatic = mkForce false;
-                source = "/state/vms/${vmName}";
+            gc.automatic = mkForce false;
-                mountPoint = "/state";
+          };
-                tag = "state";
+
-                proto = "virtiofs";
+          networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = mac;
-              }
+
-            ]
+          systemd.network.networks = {
-            # Mount persistent data from the host
+            "10-${guestCfg.networking.mainLinkName}" = {
-            ++ optional vmCfg.zfs.enable {
+              matchConfig.MACAddress = mac;
-              source = vmCfg.zfs.mountpoint;
+              DHCP = "yes";
-              mountPoint = "/persist";
+              dhcpV4Config.UseDNS = false;
-              tag = "persist";
+              dhcpV6Config.UseDNS = false;
-              proto = "virtiofs";
+              ipv6AcceptRAConfig.UseDNS = false;
+              networkConfig = {
+                IPv6PrivacyExtensions = "yes";
+                MulticastDNS = true;
+                IPv6AcceptRA = true;
+              };
+              linkConfig.RequiredForOnline = "routable";
             };
-        };
+          };
 
-        # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here
+          networking.nftables.firewall = {
-        fileSystems."/state".neededForBoot = mkForce true;
+            zones.untrusted.interfaces = [guestCfg.networking.mainLinkName];
-        fileSystems."/persist".neededForBoot = mkForce true;
-
-        # Add a writable store overlay, but since this is always ephemeral
-        # disable any store optimization from nix.
-        microvm.writableStoreOverlay = "/nix/.rw-store";
-        nix = {
-          settings.auto-optimise-store = mkForce false;
-          optimise.automatic = mkForce false;
-          gc.automatic = mkForce false;
-        };
-
-        networking.renameInterfacesByMac.${vmCfg.networking.mainLinkName} = mac;
-
-        systemd.network.networks = {
-          "10-${vmCfg.networking.mainLinkName}" = {
-            matchConfig.MACAddress = mac;
-            DHCP = "yes";
-            dhcpV4Config.UseDNS = false;
-            dhcpV6Config.UseDNS = false;
-            ipv6AcceptRAConfig.UseDNS = false;
-            networkConfig = {
-              IPv6PrivacyExtensions = "yes";
-              MulticastDNS = true;
-              IPv6AcceptRA = true;
-            };
-            linkConfig.RequiredForOnline = "routable";
           };
         };
-
-        networking.nftables.firewall = {
-          zones.untrusted.interfaces = [vmCfg.networking.mainLinkName];
-        };
       };
-    };
+
+    containers.${guestName} =
+      mkIf (guestCfg.backend == "microvm") {
+      };
   };
 in {
   imports = [
     # Add the host module, but only enable if it necessary
     inputs.microvm.nixosModules.host
     # This is opt-out, so we can't put this into the mkIf below
-    {microvm.host.enable = vms != {};}
+    {
+      microvm.host.enable =
+        any
+        (guestCfg: guestCfg.backend == "microvm")
+        (attrValues guests);
+    }
   ];
 
-  options.meta.microvms = {
+  options.node.isGuest = mkOption {
-    commonImports = mkOption {
+    type = types.bool;
-      type = types.listOf types.unspecified;
+    description = "Whether this machine is a guest on another machine.";
-      default = [];
+    default = false;
-      description = mdDoc "Modules to import on all microvms.";
-    };
-
-    networking = {
-      baseMac = mkOption {
-        type = types.net.mac;
-        description = mdDoc ''
-          This MAC address will be used as a base address to derive all MicroVM MAC addresses from.
-          A good practise is to use the physical address of the macvtap interface.
-        '';
-      };
-
-      macvtapInterface = mkOption {
-        type = types.str;
-        description = mdDoc "The macvtap interface to which MicroVMs should be attached";
-      };
-
-      wireguard = {
-        cidrv4 = mkOption {
-          type = types.net.cidrv4;
-          description = mdDoc "The ipv4 network address range to use for internal vm traffic.";
-          default = "172.31.0.0/24";
-        };
-
-        cidrv6 = mkOption {
-          type = types.net.cidrv6;
-          description = mdDoc "The ipv6 network address range to use for internal vm traffic.";
-          default = "fd00:172:31::/120";
-        };
-
-        port = mkOption {
-          default = 51829;
-          type = types.port;
-          description = mdDoc "The port to listen on.";
-        };
-
-        openFirewallRules = mkOption {
-          default = [];
-          type = types.listOf types.str;
-          description = mdDoc "The {option}`port` will be opened for all of the given rules in the nftable-firewall.";
-        };
-      };
-    };
-
-    vms = mkOption {
-      default = {};
-      description = "Defines the actual vms and handles the necessary base setup for them.";
-      type = types.attrsOf (types.submodule ({name, ...}: {
-        options = {
-          nodeName = mkOption {
-            type = types.str;
-            default = "${nodeName}-${name}";
-            description = mdDoc ''
-              The name of the resulting node. By default this will be a compound name
-              of the host's name and the vm's name to avoid name clashes. Can be
-              overwritten to designate special names to specific vms.
-            '';
-          };
-
-          networking = {
-            mainLinkName = mkOption {
-              type = types.str;
-              default = "wan";
-              description = mdDoc "The main ethernet link name inside of the VM";
-            };
-          };
-
-          zfs = {
-            enable = mkEnableOption (mdDoc "persistent data on separate zfs dataset");
-
-            pool = mkOption {
-              type = types.str;
-              description = mdDoc "The host's zfs pool on which the dataset resides";
-            };
-
-            dataset = mkOption {
-              type = types.str;
-              default = "safe/vms/${name}";
-              description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
-            };
-
-            mountpoint = mkOption {
-              type = types.str;
-              default = "/vms/${name}";
-              description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtiofs as /persist in the vm)";
-            };
-          };
-
-          autostart = mkOption {
-            type = types.bool;
-            default = false;
-            description = mdDoc "Whether this VM should be started automatically with the host";
-          };
-
-          system = mkOption {
-            type = types.str;
-            description = mdDoc "The system that this microvm should use";
-          };
-
-          modules = mkOption {
-            type = types.listOf types.unspecified;
-            default = [];
-            description = mdDoc "Additional modules to load";
-          };
-        };
-      }));
-    };
   };
 
-  config = mkIf (vms != {}) (mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList microvmConfig vms));
+  # networking = {
+  #   baseMac = mkOption {
+  #     type = types.net.mac;
+  #     description = ''
+  #       This MAC address will be used as a base address to derive all MicroVM MAC addresses from.
+  #       A good practise is to use the physical address of the macvtap interface.
+  #     '';
+  #   };
+  #
+  #   macvtapInterface = mkOption {
+  #     type = types.str;
+  #     description = "The macvtap interface to which MicroVMs should be attached";
+  #   };
+  # };
+
+  options.guests = mkOption {
+    default = {};
+    description = "Defines the actual vms and handles the necessary base setup for them.";
+    type = types.attrsOf (types.submodule ({name, ...}: {
+      options = {
+        nodeName = mkOption {
+          type = types.str;
+          default = "${nodeName}-${name}";
+          description = ''
+            The name of the resulting node. By default this will be a compound name
+            of the host's name and the vm's name to avoid name clashes. Can be
+            overwritten to designate special names to specific vms.
+          '';
+        };
+
+        backend = mkOption {
+          type = types.enum ["microvm" "container"];
+          description = ''
+            Determines how the guest will be hosted. You can currently choose
+            between microvm based deployment, or nixos containers.
+          '';
+        };
+
+        # Options for the microvm backend
+        microvm = {
+          system = mkOption {
+            type = types.str;
+            description = "The system that this microvm should use";
+          };
+        };
+
+        networking = {
+          mainLinkName = mkOption {
+            type = types.str;
+            default = "wan";
+            description = "The main ethernet link name inside of the VM";
+          };
+        };
+
+        zfs = {
+          enable = mkEnableOption "persistent data on separate zfs dataset";
+
+          pool = mkOption {
+            type = types.str;
+            description = "The host's zfs pool on which the dataset resides";
+          };
+
+          dataset = mkOption {
+            type = types.str;
+            default = "safe/guests/${name}";
+            description = "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
+          };
+
+          mountpoint = mkOption {
+            type = types.str;
+            default = "/guests/${name}";
+            description = "The host's mountpoint for the vm's dataset (will be shared via virtiofs as /persist in the vm)";
+          };
+        };
+
+        autostart = mkOption {
+          type = types.bool;
+          default = false;
+          description = "Whether this VM should be started automatically with the host";
+        };
+
+        modules = mkOption {
+          type = types.listOf types.unspecified;
+          default = [];
+          description = "Additional modules to load";
+        };
+      };
+    }));
+  };
+
+  config = mkIf (guests != {}) (mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList defineGuest guests));
 }