feat: add unified microvm & container definition; add net, misc, disko lib extension

modules/default.nix

@@ -1,7 +1,15 @@
-{
+{inputs, ...}: {
   imports = [
-    ./interface-naming.nix
+    inputs.microvm.nixosModules.host
+
     ./boot.nix
+    ./guests/default.nix
+    ./interface-naming.nix
     ./nginx.nix
+    ./node.nix
+  ];
+
+  nixpkgs.overlays = [
+    inputs.microvm.overlay
   ];
 }

modules/guests/common-guest-config.nix

@@ -0,0 +1,27 @@
+_guestName: guestCfg: {lib, ...}: let
+  inherit (lib) mkForce;
+in {
+  node.name = guestCfg.nodeName;
+  node.type = guestCfg.backend;
+
+  nix = {
+    settings.auto-optimise-store = mkForce false;
+    optimise.automatic = mkForce false;
+    gc.automatic = mkForce false;
+  };
+
+  systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = {
+    matchConfig.Name = guestCfg.networking.mainLinkName;
+    DHCP = "yes";
+    # XXX: Do we really want this?
+    dhcpV4Config.UseDNS = false;
+    dhcpV6Config.UseDNS = false;
+    ipv6AcceptRAConfig.UseDNS = false;
+    networkConfig = {
+      IPv6PrivacyExtensions = "yes";
+      MulticastDNS = true;
+      IPv6AcceptRA = true;
+    };
+    linkConfig.RequiredForOnline = "routable";
+  };
+}

modules/guests/container.nix

@@ -0,0 +1,64 @@
+guestName: guestCfg: {
+  config,
+  inputs,
+  lib,
+  pkgs,
+  ...
+}: let
+  inherit
+    (lib)
+    flip
+    mapAttrs'
+    nameValuePair
+    ;
+in {
+  ephemeral = true;
+  privateNetwork = true;
+  autoStart = guestCfg.autostart;
+  macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
+  extraFlags = [
+    "--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}"
+  ];
+  bindMounts = flip mapAttrs' guestCfg.zfs (
+    _: zfsCfg:
+      nameValuePair zfsCfg.guestMountpoint {
+        hostPath = zfsCfg.hostMountpoint;
+        isReadOnly = false;
+      }
+  );
+  nixosConfiguration = inputs.nixpkgs.lib.nixosSystem {
+    specialArgs = guestCfg.extraSpecialArgs;
+    prefix = ["nodes" "${config.node.name}-${guestName}" "config"];
+    system = null;
+    modules =
+      [
+        {
+          boot.isContainer = true;
+          networking.useHostResolvConf = false;
+
+          # We cannot force the package set via nixpkgs.pkgs and
+          # inputs.nixpkgs.nixosModules.readOnlyPkgs, since some nixosModules
+          # like nixseparatedebuginfod depend on adding packages via nixpkgs.overlays.
+          # So we just mimic the options and overlays defined by the passed pkgs set.
+          nixpkgs.hostPlatform = config.nixpkgs.hostPlatform.system;
+          nixpkgs.overlays = pkgs.overlays;
+          nixpkgs.config = pkgs.config;
+
+          # Bind the /guest/* paths from above so impermancence doesn't complain.
+          # We bind-mount stuff from the host to itself, which is perfectly defined
+          # and not recursive. This allows us to have a fileSystems entry for each
+          # bindMount which other stuff can depend upon (impermanence adds dependencies
+          # to the state fs).
+          fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg:
+            nameValuePair zfsCfg.guestMountpoint {
+              neededForBoot = true;
+              fsType = "none";
+              device = zfsCfg.guestMountpoint;
+              options = ["bind"];
+            });
+        }
+        (import ./common-guest-config.nix guestName guestCfg)
+      ]
+      ++ guestCfg.modules;
+  };
+}

modules/guests/default.nix

@@ -0,0 +1,281 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+} @ attrs: let
+  inherit
+    (lib)
+    attrNames
+    attrValues
+    attrsToList
+    disko
+    escapeShellArg
+    flatten
+    flip
+    groupBy
+    hasPrefix
+    listToAttrs
+    literalExpression
+    makeBinPath
+    mapAttrs
+    mapAttrsToList
+    mergeToplevelConfigs
+    mkIf
+    mkMerge
+    mkOption
+    net
+    types
+    ;
+
+  # All available backends
+  backends = ["microvm" "container"];
+
+  guestsByBackend =
+    lib.genAttrs backends (_: {})
+    // mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests));
+
+  # List the necessary mount units for the given guest
+  fsMountUnitsFor = guestCfg:
+    map
+    (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount")
+    (attrValues guestCfg.zfs);
+
+  # Configuration required on the host for a specific guest
+  defineGuest = _guestName: guestCfg: {
+    # Add the required datasets to the disko configuration of the machine
+    disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: {
+      ${zfsCfg.pool}.datasets.${zfsCfg.dataset} = disko.filesystem zfsCfg.hostMountpoint;
+    }));
+
+    # Ensure that the zfs dataset exists before it is mounted.
+    systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let
+      fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
+    in {
+      "zfs-ensure-${utils.escapeSystemdPath zfsCfg.hostMountpoint}" = {
+        wantedBy = [fsMountUnit];
+        before = [fsMountUnit];
+        after = [
+          "zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service"
+          "zfs-mount.target"
+        ];
+        unitConfig.DefaultDependencies = "no";
+        serviceConfig.Type = "oneshot";
+        script = let
+          poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}";
+          diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset};
+        in ''
+          export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
+          if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
+            ${diskoDataset._create}
+          fi
+        '';
+      };
+    }));
+  };
+
+  defineMicrovm = guestName: guestCfg: {
+    # Ensure that the zfs dataset exists before it is mounted.
+    systemd.services."microvm@${guestName}" = {
+      requires = fsMountUnitsFor guestCfg;
+      after = fsMountUnitsFor guestCfg;
+    };
+
+    microvm.vms.${guestName} = import ./microvm.nix guestName guestCfg attrs;
+  };
+
+  defineContainer = guestName: guestCfg: {
+    # Ensure that the zfs dataset exists before it is mounted.
+    systemd.services."container@${guestName}" = {
+      requires = fsMountUnitsFor guestCfg;
+      after = fsMountUnitsFor guestCfg;
+      # Don't use the notify service type. Using exec will always consider containers
+      # started immediately and donesn't wait until the container is fully booted.
+      # Containers should behave like independent machines, and issues inside the container
+      # will unnecessarily lock up the service on the host otherwise.
+      # This causes issues on system activation or when containers take longer to start
+      # than TimeoutStartSec.
+      serviceConfig.Type = lib.mkForce "exec";
+    };
+
+    containers.${guestName} = import ./container.nix guestName guestCfg attrs;
+  };
+in {
+  imports = [
+    {
+      # This is opt-out, so we can't put this into the mkIf below
+      microvm.host.enable = guestsByBackend.microvm != {};
+    }
+  ];
+
+  options.node.type = mkOption {
+    type = types.enum (["host"] ++ backends);
+    description = "The type of this machine.";
+    default = "host";
+  };
+
+  options.containers = mkOption {
+    type = types.attrsOf (types.submodule (submod: {
+      options.nixosConfiguration = mkOption {
+        type = types.unspecified;
+        default = null;
+        description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
+      };
+      config = mkIf (submod.config.nixosConfiguration != null) {
+        path = submod.config.nixosConfiguration.config.system.build.toplevel;
+      };
+    }));
+  };
+
+  options.guests = mkOption {
+    default = {};
+    description = "Defines the actual vms and handles the necessary base setup for them.";
+    type = types.attrsOf (types.submodule (submod: {
+      options = {
+        nodeName = mkOption {
+          type = types.str;
+          default = "${config.node.name}-${submod.config._module.args.name}";
+          description = ''
+            The name of the resulting node. By default this will be a compound name
+            of the host's name and the guest's name to avoid name clashes. Can be
+            overwritten to designate special names to specific guests.
+          '';
+        };
+
+        backend = mkOption {
+          type = types.enum backends;
+          description = ''
+            Determines how the guest will be hosted. You can currently choose
+            between microvm based deployment, or nixos containers.
+          '';
+        };
+
+        extraSpecialArgs = mkOption {
+          type = types.attrs;
+          default = {};
+          example = literalExpression "{ inherit inputs; }";
+          description = ''
+            Extra `specialArgs` passed to each guest system definition. This
+            option can be used to pass additional arguments to all modules.
+          '';
+        };
+
+        # Options for the microvm backend
+        microvm = {
+          system = mkOption {
+            type = types.str;
+            description = "The system that this microvm should use";
+          };
+
+          macvtap = mkOption {
+            type = types.str;
+            description = "The host interface to which the microvm should be attached via macvtap";
+          };
+
+          baseMac = mkOption {
+            type = types.net.mac;
+            description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
+            default = "02:01:27:00:00:00";
+          };
+
+          mac = mkOption {
+            type = types.net.mac;
+            description = "The MAC address for the guest's macvtap interface";
+            default = let
+              base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
+            in
+              (net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name};
+          };
+        };
+
+        # Options for the container backend
+        container = {
+          macvlan = mkOption {
+            type = types.str;
+            description = "The host interface to which the container should be attached";
+          };
+        };
+
+        networking.mainLinkName = mkOption {
+          type = types.str;
+          description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host.";
+          default =
+            if submod.config.backend == "microvm"
+            then submod.config.microvm.macvtap
+            else if submod.config.backend == "container"
+            then "mv-${submod.config.container.macvlan}"
+            else throw "Invalid backend";
+        };
+
+        zfs = mkOption {
+          description = "zfs datasets to mount into the guest";
+          default = {};
+          type = types.attrsOf (types.submodule (zfsSubmod: {
+            options = {
+              pool = mkOption {
+                type = types.str;
+                description = "The host's zfs pool on which the dataset resides";
+              };
+
+              dataset = mkOption {
+                type = types.str;
+                example = "safe/guests/mycontainer";
+                description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
+              };
+
+              hostMountpoint = mkOption {
+                type = types.path;
+                default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}";
+                example = "/guests/mycontainer/persist";
+                description = "The host's mountpoint for the guest's dataset";
+              };
+
+              guestMountpoint = mkOption {
+                type = types.path;
+                default = zfsSubmod.config._module.args.name;
+                example = "/persist";
+                description = "The mountpoint inside the guest.";
+              };
+            };
+          }));
+        };
+
+        autostart = mkOption {
+          type = types.bool;
+          default = false;
+          description = "Whether this guest should be started automatically with the host";
+        };
+
+        modules = mkOption {
+          type = types.listOf types.unspecified;
+          default = [];
+          description = "Additional modules to load";
+        };
+      };
+    }));
+  };
+
+  config = mkIf (config.guests != {}) (
+    mkMerge [
+      {
+        systemd.tmpfiles.rules = [
+          "d /guests 0700 root root -"
+        ];
+
+        assertions = flatten (flip mapAttrsToList config.guests (
+          guestName: guestCfg:
+            flip mapAttrsToList guestCfg.zfs (
+              zfsName: zfsCfg: {
+                assertion = hasPrefix "/" zfsCfg.guestMountpoint;
+                message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path.";
+              }
+            )
+        ));
+      }
+      (mergeToplevelConfigs ["disko" "systemd"] (mapAttrsToList defineGuest config.guests))
+      (mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container))
+      (mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm))
+    ]
+  );
+}

modules/guests/microvm.nix

@@ -0,0 +1,82 @@
+guestName: guestCfg: {
+  config,
+  inputs,
+  lib,
+  pkgs,
+  ...
+}: let
+  inherit
+    (lib)
+    flip
+    mapAttrsToList
+    mkDefault
+    mkForce
+    ;
+in {
+  specialArgs = guestCfg.extraSpecialArgs;
+  pkgs = inputs.self.pkgs.${guestCfg.microvm.system};
+  inherit (guestCfg) autostart;
+  config = {
+    imports =
+      guestCfg.modules
+      ++ [
+        (import ./common-guest-config.nix guestName guestCfg)
+        ({config, ...}: {
+          # Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
+          boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"];
+        })
+      ];
+
+    # TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
+    environment.noXlibs = mkForce false;
+    lib.microvm.mac = guestCfg.microvm.mac;
+
+    microvm = {
+      hypervisor = mkDefault "qemu";
+
+      # Give them some juice by default
+      mem = mkDefault (1024 + 2048);
+
+      # Add a writable store overlay, but since this is always ephemeral
+      # disable any store optimization from nix.
+      writableStoreOverlay = "/nix/.rw-store";
+
+      # MACVTAP bridge to the host's network
+      interfaces = [
+        {
+          type = "macvtap";
+          id = "vm-${guestName}";
+          inherit (guestCfg.microvm) mac;
+          macvtap = {
+            link = guestCfg.microvm.macvtap;
+            mode = "bridge";
+          };
+        }
+      ];
+
+      shares =
+        [
+          # Share the nix-store of the host
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "ro-store";
+            proto = "virtiofs";
+          }
+        ]
+        ++ flip mapAttrsToList guestCfg.zfs (
+          _: zfsCfg: {
+            source = zfsCfg.hostMountpoint;
+            mountPoint = zfsCfg.guestMountpoint;
+            tag = builtins.substring 0 16 (builtins.hashString "sha256" zfsCfg.hostMountpoint);
+            proto = "virtiofs";
+          }
+        );
+    };
+
+    networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac;
+    systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce {
+      MACAddress = guestCfg.microvm.mac;
+    };
+  };
+}