feat: add macvtap networking to microvms

README.md

@@ -67,6 +67,7 @@ This is my personal nix config.
 - (Optional) ssh into the target (keys are already set up)
 - Run `install-system` and reboot
 - Retrieve the new host identity by using `ssh-keyscan <host/ip> | grep -o 'ed25519.*' > host/<host>/secrets/host.pub`
+- (If the host has microvms, also retrieve their identities!)
 - Rekey the secrets for the new identity `nix run .#rekey`
 - Deploy again remotely via colmena
 

hosts/common/core/system.nix

@@ -12,10 +12,21 @@
     libWithNet = (import "${inputs.lib-net}/net.nix" {inherit lib;}).lib;
   in
     lib.recursiveUpdate libWithNet {
-      net.cidr = rec {
+      net = {
-        hostCidr = n: x: "${libWithNet.net.cidr.host n x}/${libWithNet.net.cidr.length x}";
+        cidr = rec {
-        ip = x: lib.head (lib.splitString "/" x);
+          hostCidr = n: x: "${libWithNet.net.cidr.host n x}/${libWithNet.net.cidr.length x}";
-        canonicalize = x: libWithNet.net.cidr.make (libWithNet.net.cidr.length x) (ip x);
+          ip = x: lib.head (lib.splitString "/" x);
+          canonicalize = x: libWithNet.net.cidr.make (libWithNet.net.cidr.length x) (ip x);
+        };
+        mac = {
+          # Adds offset to the given base address and ensures the result is in
+          # a locally administered range by replacing the second nibble with a 2.
+          addPrivate = base: offset: let
+            added = libWithNet.net.mac.add base offset;
+            pre = lib.substring 0 1 added;
+            suf = lib.substring 2 (-1) added;
+          in "${pre}2${suf}";
+        };
       };
     };
 

hosts/ward/default.nix

@@ -3,6 +3,7 @@
   inputs,
   lib,
   nixos-hardware,
+  nodeSecrets,
   pkgs,
   ...
 }: {
@@ -25,7 +26,16 @@
 
   boot.initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" "sdhci_pci" "r8169"];
 
-  extra.microvms.test.system = "x86_64-linux";
+  extra.microvms = let
+    macOffset = config.lib.net.mac.addPrivate nodeSecrets.networking.interfaces."wan-nic".mac;
+  in {
+    test = {
+      autostart = true;
+      mac = macOffset "00:00:00:00:00:01";
+      macvtap = "wan";
+      system = "x86_64-linux";
+    };
+  };
 
   #services.authelia.instances.main = {
   #  enable = true;

hosts/ward/net.nix

@@ -13,7 +13,30 @@ in {
 
   boot.initrd.systemd.network = {
     enable = true;
-    networks = {inherit (config.systemd.network.networks) "10-wan";};
+    networks."10-wan" = {
+      DHCP = "yes";
+      #address = [
+      #  "192.168.178.2/24"
+      #  "fd00::1/64"
+      #];
+      #gateway = [
+      #];
+      matchConfig.MACAddress = nodeSecrets.networking.interfaces."wan-nic".mac;
+      networkConfig.IPv6PrivacyExtensions = "kernel";
+      dhcpV4Config.RouteMetric = 20;
+      dhcpV6Config.RouteMetric = 20;
+    };
+  };
+
+  systemd.network.netdevs."10-wan" = {
+    netdevConfig = {
+      Name = "wan";
+      Kind = "macvtap";
+    };
+    extraConfig = ''
+      [MACVTAP]
+      Mode=bridge
+    '';
   };
 
   systemd.network.networks = {
@@ -27,7 +50,14 @@ in {
       dhcpV4Config.RouteMetric = 10;
       dhcpV6Config.RouteMetric = 10;
     };
-    "10-wan" = {
+    "10-wan-nic" = {
+      matchConfig.MACAddress = nodeSecrets.networking.interfaces."wan-nic".mac;
+      extraConfig = ''
+        [Network]
+        MACVTAP=wan
+      '';
+    };
+    "11-wan" = {
       DHCP = "yes";
       #address = [
       #  "192.168.178.2/24"
@@ -35,7 +65,7 @@ in {
       #];
       #gateway = [
       #];
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.wan.mac;
+      matchConfig.Name = "wan";
       networkConfig.IPv6PrivacyExtensions = "kernel";
       dhcpV4Config.RouteMetric = 20;
       dhcpV6Config.RouteMetric = 20;

modules/microvms.nix

@@ -1,16 +1,21 @@
 {
-  inputs,
   config,
+  extraLib,
+  inputs,
   lib,
+  microvm,
   nodeName,
   nodePath,
-  microvm,
   pkgs,
   ...
 }: let
   inherit
     (lib)
+    attrNames
+    concatStringsSep
+    filterAttrs
     mapAttrs
+    mapAttrsToList
     mdDoc
     mkDefault
     mkForce
@@ -40,9 +45,9 @@
         interfaces = [
           {
             type = "macvtap";
-            id = "macvtap1";
+            id = "vm-${vmName}";
             macvtap = {
-              link = "wan0";
+              link = vmCfg.macvtap;
               mode = "bridge";
             };
             inherit (vmCfg) mac;
@@ -79,37 +84,46 @@ in {
     description = "Provides a base configuration for MicroVMs.";
     type = types.attrsOf (types.submodule {
       options = {
-        system = mkOption {
+        autostart = mkOption {
-          type = types.str;
+          type = types.bool;
-          description = mdDoc "The system that this microvm should use";
+          default = false;
+          description = mdDoc "Whether this VM should be started automatically with the host";
         };
 
         mac = mkOption {
-          type = types.mac;
+          type = config.lib.net.types.mac;
           description = mdDoc "The MAC address to assign to this VM";
         };
+
+        macvtap = mkOption {
+          type = types.str;
+          description = mdDoc "The macvtap interface to attach to";
+        };
+
+        system = mkOption {
+          type = types.str;
+          description = mdDoc "The system that this microvm should use";
+        };
       };
     });
   };
 
   config = {
+    assertions = let
+      duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
+    in [
+      {
+        assertion = duplicateMacs == [];
+        message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
+      }
+    ];
+
     microvm = {
       host.enable = cfg != {};
       declarativeUpdates = true;
       restartIfChanged = true;
       vms = mkIf (cfg != {}) (mapAttrs defineMicrovm cfg);
+      autostart = mkIf (cfg != {}) (attrNames (filterAttrs (_: v: v.autostart) cfg));
     };
-
-    #systemd.network.netdevs.jrsptap = {
-    #  netdevConfig = {
-    #    Name = "jrsptap";
-    #    Kind = "macvtap";
-    #    MACAddress = "...";
-    #  };
-    #};
-    #systemd.network.networks.vms = {
-    #  matchConfig.Name = "vms";
-    #  networkConfig.MACVTAP = "jrsptap";
-    #};
   };
 }

nix/lib.nix

@@ -37,7 +37,7 @@ in rec {
   in
     foldl' addOrUpdate {} xs;
 
-  # Returns all elements in xs that occur at least once
+  # Returns all elements in xs that occur at least twice
   duplicates = xs: let
     occurrences = countOccurrences xs;
   in