feat: wip: add container backend to guests

README.md

@@ -30,7 +30,7 @@ Server related stuff:
   - [loki](https://github.com/grafana/loki) and [promtail](https://grafana.com/docs/loki/latest/clients/promtail/) for logs
 - Single-Sign-On for all services using oauth2 via [kanidm](https://github.com/kanidm/kanidm)
 - Zoned nftables firewall via [nixos-nftables-firewall](https://github.com/thelegy/nixos-nftables-firewall)
-- Service isolation using [microvms](https://github.com/astro/microvm.nix) <!-- XXX: where possible, otherwise oci-containers -->
+- Service isolation using nixos-containers and [microvms](https://github.com/astro/microvm.nix)
 <!--
 XXX: todo, use details summary to show gallery of services
 
@@ -43,7 +43,7 @@ XXX: todo, use details summary to show gallery of services
 ---|---|---|---
 💻 | nom | Gigabyte AERO 15-W8 (i7-8750H) | My laptop and my main portable development machine <sub>Framework when?</sub>
 🖥️ | kroma | PC (AMD Ryzen 9 5900X) | Main workstation and development machine, also for some occasional gaming
-🖥️ | ward | ODROID H3 | Energy efficient SBC for my home firewall and some lightweight services using microvms.
+🖥️ | ward | ODROID H3 | Energy efficient SBC for my home firewall and some lightweight services using containers and microvms.
 🥔 | zackbiene | ODROID N2+ | ARM SBC for home automation, isolating the sketchy stuff from my main network
 ☁️  | envoy | Hetzner Cloud server | Mailserver
 ☁️  | sentinel | Hetzner Cloud server | Proxies and protects my local services
@@ -123,7 +123,7 @@ Afterwards:
 
 - Run `install-system` in the live environment and reboot
 - Retrieve the new host identity by using `ssh-keyscan <host/ip> | grep -o 'ssh-ed25519.*' > hosts/<host>/secrets/host.pub`
-- (If the host has microvms, also retrieve their identities!)
+- (If the host has guests, also retrieve their identities!)
 - Rekey the secrets for the new identity `nix run .#rekey`
 - Deploy again
 

STRUCTURE.md

@@ -13,8 +13,8 @@ Make sure to utilize the github search if you know what you need!
   - `host.pub` This host's public key (retrieved after initial setup). Used to rekey secrets so the host can access them at runtime.
   - `local.nix.age` Repository-wide local secrets. Decrypted on import, see `modules/repo/secrets.nix` for more information.
 
-  Some hosts define microvms that run as virtualized guests. Their configuration is usually just a single file
+  Some hosts define guests that run as containerized or virtualized guests. Their configuration is usually just a single file
-  stored in `microvms/<vm>.nix`. Their secrets are usually stored in a subfolder of the host's secrets.
+  stored in `guests/<name>.nix`. Their secrets are usually stored in a subfolder of the host's secrets folder.
 
 - `lib/` contains extra library functions that are needed throughout the config.
 

flake.nix

@@ -141,14 +141,14 @@
       inherit
         (import ./nix/hosts.nix inputs)
         hosts
-        microvmConfigurations
+        guestConfigs
         nixosConfigurations
         nixosConfigurationsMinimal
         ;
 
       # All nixosSystem instanciations are collected here, so that we can refer
       # to any system via nodes.<name>
-      nodes = self.nixosConfigurations // self.microvmConfigurations;
+      nodes = self.nixosConfigurations // self.guestConfigs;
       # Add a shorthand to easily target toplevel derivations
       "@" = mapAttrs (_: v: v.config.system.build.toplevel) self.nodes;
 

hosts/ward/default.nix

@@ -43,17 +43,13 @@
   # TODO track my github stats
   # services.telegraf.extraConfig.inputs.github = {};
 
-  meta.microvms.commonImports = [
-    ../../modules
-    ./microvms/common.nix
-  ];
-
   #guests.adguardhome = {
   #  backend = "microvm";
   #  microvm = {
   #    system = "x86_64-linux";
-  #    autostart = true;
+  #    macvtapInterface = "lan";
   #  };
+  #  autostart = true;
   #  zfs = {
   #    enable = true;
   #    pool = "rpool";
@@ -62,53 +58,47 @@
   #};
 
   guests = let
-    mkMicrovm = system: module: {
+    mkGuest = mainModule: {
-      backend = "microvm";
-      microvm = {
-        system = "x86_64-linux";
       autostart = true;
-      };
       zfs = {
         enable = true;
         pool = "rpool";
       };
       modules = [
         ../../modules
-        module
+        ./guests/common.nix
+        ({config, ...}: {node.secretsDir = ./secrets + "/${config.node.name}";})
+        mainModule
       ];
     };
-  in {
-    adguardhome = mkMicrovm "x86_64-linux" ./guests/adguardhome.nix;
-  };
 
-  meta.microvms.vms = let
+    mkMicrovm = system: mainModule:
-    defaultConfig = name: {
+      mkGuest mainModule
+      // {
+        backend = "microvm";
+        microvm = {
           system = "x86_64-linux";
-      autostart = true;
+          macvtapInterface = "lan";
-      zfs = {
-        enable = true;
-        pool = "rpool";
         };
-      modules = [
+      };
-        # XXX: this could be interpolated in-place but statix has a bug https://github.com/nerdypepper/statix/issues/75
+
-        (./microvms + "/${name}.nix")
+    mkContainer = mainModule:
-        {node.secretsDir = ./secrets + "/${name}";}
+      mkGuest mainModule
-      ];
+      // {
+        backend = "container";
+        container.macvlan = "lan";
       };
   in
-    lib.mkIf (!minimal) (
+    lib.mkIf (!minimal) {
-      lib.genAttrs [
+      adguardhome = mkContainer ./guests/adguardhome.nix;
-        "adguardhome"
+      forgejo = mkContainer ./guests/forgejo.nix;
-        "forgejo"
+      grafana = mkContainer ./guests/grafana.nix;
-        "grafana"
+      influxdb = mkContainer ./guests/influxdb.nix;
-        "influxdb"
+      kanidm = mkContainer ./guests/kanidm.nix;
-        "kanidm"
+      loki = mkContainer ./guests/loki.nix;
-        "loki"
+      paperless = mkContainer ./guests/paperless.nix;
-        "paperless"
+      vaultwarden = mkContainer ./guests/vaultwarden.nix;
-        "vaultwarden"
+    };
-      ]
-      defaultConfig
-    );
 
   #ddclient = defineVm;
   #samba+wsdd = defineVm;

hosts/ward/guests/paperless.nix

@@ -6,9 +6,8 @@
   sentinelCfg = nodes.sentinel.config;
   paperlessDomain = "paperless.${sentinelCfg.repo.secrets.local.personalDomain}";
 in {
-  microvm.mem = 1024 * 12;
+  # XXX: remove microvm.mem = 1024 * 12;
-  # XXX: increase once real hardware is used
+  # XXX: remove microvm.vcpu = 4;
-  microvm.vcpu = 4;
 
   meta.wireguard-proxy.sentinel.allowedTCPPorts = [
     config.services.paperless.port

hosts/ward/kea.nix

@@ -45,12 +45,12 @@ in {
               data = net.cidr.host 1 lanCidrv4;
             }
           ];
-          reservations = [
+          # TODO reservations = [
-            {
+          # TODO   {
-              hw-address = nodes.ward-adguardhome.config.lib.microvm.mac;
+          # TODO     hw-address = nodes.ward-adguardhome.config.lib.microvm.mac;
-              ip-address = dnsIp;
+          # TODO     ip-address = dnsIp;
-            }
+          # TODO   }
-          ];
+          # TODO ];
         }
       ];
     };

hosts/ward/net.nix

@@ -124,11 +124,6 @@ in {
     };
   };
 
-  meta.microvms.networking = {
-    baseMac = config.repo.secrets.local.networking.interfaces.lan.mac;
-    macvtapInterface = "lan";
-  };
-
   # Allow accessing influx
   meta.wireguard.proxy-sentinel.client.via = "sentinel";
 }

modules/config/issue.nix

@@ -9,6 +9,6 @@
       ''This is \e{cyan}\n\e{reset} [\e{lightblue}\l\e{reset}] (\s \m \r)''
     ]
     # Disabled for guests because of frequent redraws (-> pushed to syslog on the host)
-    ++ lib.optional (!config.guests.isGuest) ''\e{halfbright}\4\e{reset} \e{halfbright}\6\e{reset}''
+    ++ lib.optional (config.node.type == "host") ''\e{halfbright}\4\e{reset} \e{halfbright}\6\e{reset}''
     ++ [""]);
 }

modules/default.nix

@@ -25,8 +25,9 @@
     ./config/system.nix
     ./config/users.nix
 
+    ./guests
+
     ./meta/kanidm.nix
-    ./meta/microvms.nix
     ./meta/nginx.nix
     ./meta/oauth2-proxy.nix
     ./meta/promtail.nix

modules/guests/common-guest-config.nix

@@ -0,0 +1,31 @@
+_guestName: guestCfg: {lib, ...}: let
+  inherit (lib) mkForce;
+in {
+  node.name = guestCfg.nodeName;
+  node.type = guestCfg.backend;
+
+  nix = {
+    settings.auto-optimise-store = mkForce false;
+    optimise.automatic = mkForce false;
+    gc.automatic = mkForce false;
+  };
+
+  systemd.network.networks = {
+    "10-${guestCfg.networking.mainLinkName}" = {
+      DHCP = "yes";
+      dhcpV4Config.UseDNS = false;
+      dhcpV6Config.UseDNS = false;
+      ipv6AcceptRAConfig.UseDNS = false;
+      networkConfig = {
+        IPv6PrivacyExtensions = "yes";
+        MulticastDNS = true;
+        IPv6AcceptRA = true;
+      };
+      linkConfig.RequiredForOnline = "routable";
+    };
+  };
+
+  networking.nftables.firewall = {
+    zones.untrusted.interfaces = [guestCfg.networking.mainLinkName];
+  };
+}

modules/guests/container.nix

@@ -0,0 +1,29 @@
+guestName: guestCfg: {
+  config,
+  lib,
+  ...
+} @ attrs: let
+  inherit (lib) mkMerge;
+in {
+  autoStart = guestCfg.autostart;
+  specialArgs =
+    attrs
+    // {
+      parentNode = config;
+    };
+  macvlans = [guestCfg.container.macvlan];
+  ephemeral = true;
+  privateNetwork = true;
+  config = mkMerge (guestCfg.modules
+    ++ [
+      (import ./common-guest-config.nix guestName guestCfg)
+      {
+        systemd.network.networks = {
+          "10-${guestCfg.networking.mainLinkName}" = {
+            matchConfig.OriginalName = "mv-${guestCfg.container.macvlan}";
+            linkConfig.Name = guestCfg.networking.mainLinkName;
+          };
+        };
+      }
+    ]);
+}

modules/guests/default.nix

@@ -4,12 +4,10 @@
   lib,
   pkgs,
   utils,
-  minimal,
   ...
-}: let
+} @ attrs: let
   inherit
     (lib)
-    attrNames
     attrValues
     any
     disko
@@ -17,19 +15,13 @@
     makeBinPath
     mapAttrsToList
     mergeToplevelConfigs
-    mkDefault
     mkEnableOption
-    mkForce
     mkIf
     mkOption
-    net
-    optional
     types
     ;
 
-  cfg = config.guests;
   nodeName = config.node.name;
-  inherit (cfg) guests;
 
   # Configuration required on the host for a specific guest
   defineGuest = guestName: guestCfg: {
@@ -79,115 +71,18 @@
           requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
           after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
         };
-      };
 
-    microvm.vms.${guestName} = let
+        "container@${guestName}" = mkIf (guestCfg.backend == "container") {
-      mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames guests)).${guestName};
+          requires = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
-    in
+          after = [fsMountUnit "zfs-chown-${utils.escapeSystemdPath guestCfg.zfs.mountpoint}.service"];
-      mkIf (guestCfg.backend == "microvm") {
-        # Allow children microvms to know which node is their parent
-        specialArgs = {
-          parentNode = config;
-          inherit (inputs.self) nodes;
-          inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib;
-          inherit inputs;
-          inherit minimal;
-        };
-        pkgs = inputs.self.pkgs.${guestCfg.microvm.system};
-        inherit (guestCfg) autostart;
-        config = {
-          imports = guestCfg.modules;
-          node.name = guestCfg.nodeName;
-          node.isGuest = true;
-
-          # TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
-          environment.noXlibs = mkForce false;
-          lib.microvm.mac = mac;
-
-          microvm = {
-            hypervisor = mkDefault "qemu";
-
-            # Give them some juice by default
-            mem = mkDefault (2 * 1024);
-
-            # MACVTAP bridge to the host's network
-            interfaces = [
-              {
-                type = "macvtap";
-                id = "vm-${guestName}";
-                inherit mac;
-                macvtap = {
-                  link = cfg.networking.macvtapInterface;
-                  mode = "bridge";
-                };
-              }
-            ];
-
-            shares =
-              [
-                # Share the nix-store of the host
-                {
-                  source = "/nix/store";
-                  mountPoint = "/nix/.ro-store";
-                  tag = "ro-store";
-                  proto = "virtiofs";
-                }
-                {
-                  source = "/state/guests/${guestName}";
-                  mountPoint = "/state";
-                  tag = "state";
-                  proto = "virtiofs";
-                }
-              ]
-              # Mount persistent data from the host
-              ++ optional guestCfg.zfs.enable {
-                source = guestCfg.zfs.mountpoint;
-                mountPoint = "/persist";
-                tag = "persist";
-                proto = "virtiofs";
         };
       };
 
-          # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here
+    microvm.vms.${guestName} =
-          fileSystems."/state".neededForBoot = mkForce true;
+      mkIf (guestCfg.backend == "microvm") (import ./microvm.nix guestName guestCfg attrs);
-          fileSystems."/persist".neededForBoot = mkForce true;
-
-          # Add a writable store overlay, but since this is always ephemeral
-          # disable any store optimization from nix.
-          microvm.writableStoreOverlay = "/nix/.rw-store";
-          nix = {
-            settings.auto-optimise-store = mkForce false;
-            optimise.automatic = mkForce false;
-            gc.automatic = mkForce false;
-          };
-
-          networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = mac;
-
-          systemd.network.networks = {
-            "10-${guestCfg.networking.mainLinkName}" = {
-              matchConfig.MACAddress = mac;
-              DHCP = "yes";
-              dhcpV4Config.UseDNS = false;
-              dhcpV6Config.UseDNS = false;
-              ipv6AcceptRAConfig.UseDNS = false;
-              networkConfig = {
-                IPv6PrivacyExtensions = "yes";
-                MulticastDNS = true;
-                IPv6AcceptRA = true;
-              };
-              linkConfig.RequiredForOnline = "routable";
-            };
-          };
-
-          networking.nftables.firewall = {
-            zones.untrusted.interfaces = [guestCfg.networking.mainLinkName];
-          };
-        };
-      };
 
     containers.${guestName} =
-      mkIf (guestCfg.backend == "microvm") {
+      mkIf (guestCfg.backend == "container") (import ./container.nix guestName guestCfg attrs);
-      };
   };
 in {
   imports = [
@@ -198,31 +93,16 @@ in {
       microvm.host.enable =
         any
         (guestCfg: guestCfg.backend == "microvm")
-        (attrValues guests);
+        (attrValues config.guests);
     }
   ];
 
-  options.node.isGuest = mkOption {
+  options.node.type = mkOption {
-    type = types.bool;
+    type = types.enum ["host" "microvm" "container"];
-    description = "Whether this machine is a guest on another machine.";
+    description = "The type of this machine.";
-    default = false;
+    default = "host";
   };
 
-  # networking = {
-  #   baseMac = mkOption {
-  #     type = types.net.mac;
-  #     description = ''
-  #       This MAC address will be used as a base address to derive all MicroVM MAC addresses from.
-  #       A good practise is to use the physical address of the macvtap interface.
-  #     '';
-  #   };
-  #
-  #   macvtapInterface = mkOption {
-  #     type = types.str;
-  #     description = "The macvtap interface to which MicroVMs should be attached";
-  #   };
-  # };
-
   options.guests = mkOption {
     default = {};
     description = "Defines the actual vms and handles the necessary base setup for them.";
@@ -252,6 +132,19 @@ in {
             type = types.str;
             description = "The system that this microvm should use";
           };
+
+          macvtapInterface = mkOption {
+            type = types.str;
+            description = "The host macvtap interface to which the microvm should be attached";
+          };
+        };
+
+        # Options for the container backend
+        container = {
+          macvlan = mkOption {
+            type = types.str;
+            description = "The host interface to which the container should be attached";
+          };
         };
 
         networking = {
@@ -298,5 +191,7 @@ in {
     }));
   };
 
-  config = mkIf (guests != {}) (mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList defineGuest guests));
+  config =
+    mkIf (config.guests != {})
+    (mergeToplevelConfigs ["containers" "disko" "microvm" "systemd"] (mapAttrsToList defineGuest config.guests));
 }

modules/guests/microvm.nix

@@ -0,0 +1,101 @@
+guestName: guestCfg: {
+  config,
+  inputs,
+  lib,
+  pkgs,
+  minimal,
+  ...
+}: let
+  inherit
+    (lib)
+    attrNames
+    mkDefault
+    mkForce
+    net
+    optional
+    ;
+
+  mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames config.guests)).${guestName};
+in {
+  specialArgs = {
+    parentNode = config;
+    inherit (inputs.self) nodes;
+    inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib;
+    inherit inputs;
+    inherit minimal;
+  };
+  pkgs = inputs.self.pkgs.${guestCfg.microvm.system};
+  inherit (guestCfg) autostart;
+  config = {
+    imports = guestCfg.modules ++ [(import ./common-guest-config.nix guestName guestCfg)];
+
+    # TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
+    environment.noXlibs = mkForce false;
+    lib.microvm.mac = mac;
+
+    microvm = {
+      hypervisor = mkDefault "qemu";
+
+      # Give them some juice by default
+      mem = mkDefault (2 * 1024);
+
+      # MACVTAP bridge to the host's network
+      interfaces = [
+        {
+          type = "macvtap";
+          id = "vm-${guestName}";
+          inherit mac;
+          macvtap = {
+            link = guestCfg.microvm.macvtapInterface;
+            mode = "bridge";
+          };
+        }
+      ];
+
+      shares =
+        [
+          # Share the nix-store of the host
+          {
+            source = "/nix/store";
+            mountPoint = "/nix/.ro-store";
+            tag = "ro-store";
+            proto = "virtiofs";
+          }
+          {
+            source = "/state/guests/${guestName}";
+            mountPoint = "/state";
+            tag = "state";
+            proto = "virtiofs";
+          }
+        ]
+        # Mount persistent data from the host
+        ++ optional guestCfg.zfs.enable {
+          source = guestCfg.zfs.mountpoint;
+          mountPoint = "/persist";
+          tag = "persist";
+          proto = "virtiofs";
+        };
+    };
+
+    # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here
+    fileSystems."/state".neededForBoot = mkForce true;
+    fileSystems."/persist".neededForBoot = mkForce true;
+
+    # Add a writable store overlay, but since this is always ephemeral
+    # disable any store optimization from nix.
+    microvm.writableStoreOverlay = "/nix/.rw-store";
+    nix = {
+      settings.auto-optimise-store = mkForce false;
+      optimise.automatic = mkForce false;
+      gc.automatic = mkForce false;
+    };
+
+    networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = mac;
+
+    systemd.network.networks = {
+      "10-${guestCfg.networking.mainLinkName}" = {
+        matchConfig.MACAddress = mac;
+      };
+    };
+  };
+}

nix/hosts.nix

@@ -48,17 +48,24 @@ inputs: let
   nixosConfigurations = flip mapAttrs nixosHosts (mkHost {minimal = false;});
   nixosConfigurationsMinimal = flip mapAttrs nixosHosts (mkHost {minimal = true;});
 
-  # True NixOS nodes can define additional microvms (guest nodes) that are built
+  # True NixOS nodes can define additional guest nodes that are built
-  # together with it. We collect all defined microvm nodes from each node here
+  # together with it. We collect all defined guests from each node here
   # to allow accessing any node via the unified attribute `nodes`.
-  microvmConfigurations = flip concatMapAttrs self.nixosConfigurations (_: node:
+  guestConfigs = flip concatMapAttrs self.nixosConfigurations (_: node:
-    mapAttrs'
+    flip mapAttrs' (node.config.guests or {}) (guestName: guestDef:
-    (vm: def: nameValuePair def.nodeName node.config.microvm.vms.${vm}.config)
+      nameValuePair guestDef.nodeName (
-    (node.config.meta.microvms.vms or {}));
+        if guestDef.backend == "microvm"
+        then node.config.microvm.vms.${guestName}.config
+        else {
+          # We can only access the .config part of nixosSystem here unfortunately,
+          # since the rest is not exposed by the nixos module.
+          inherit (node.config.containers.${guestName}) config;
+        }
+      )));
 in {
   inherit
     hosts
-    microvmConfigurations
+    guestConfigs
     nixosConfigurations
     nixosConfigurationsMinimal
     ;