feat: use own nixosSystem invocation for containers, add bind mounts

modules/config/secrets.nix

@@ -54,6 +54,8 @@
   # to create a link called /run/agenix. Agenix should probably fail in this case,
   # but doesn't and instead puts the generation link into the existing directory.
   # TODO See https://github.com/ryantm/agenix/pull/187.
-  system.activationScripts.removeAgenixLink.text = "[[ ! -L /run/agenix ]] && [[ -d /run/agenix ]] && rm -rf /run/agenix";
+  system.activationScripts = lib.mkIf (config.age.secrets != {}) {
-  system.activationScripts.agenixNewGeneration.deps = ["removeAgenixLink"];
+    removeAgenixLink.text = "[[ ! -L /run/agenix ]] && [[ -d /run/agenix ]] && rm -rf /run/agenix";
+    activationScripts.agenixNewGeneration.deps = ["removeAgenixLink"];
+  };
 }

modules/default.nix

@@ -27,6 +27,7 @@
 
     ./guests
 
+    # TODO merge as ./*
     ./meta/kanidm.nix
     ./meta/nginx.nix
     ./meta/oauth2-proxy.nix

modules/guests/container.nix

@@ -1,29 +1,66 @@
 guestName: guestCfg: {
   config,
+  inputs,
   lib,
+  minimal,
+  nodes,
+  pkgs,
   ...
-} @ attrs: let
+}: {
-  inherit (lib) mkMerge;
-in {
   autoStart = guestCfg.autostart;
-  specialArgs =
+  macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
-    attrs
-    // {
-      parentNode = config;
-    };
-  macvlans = [guestCfg.container.macvlan];
   ephemeral = true;
   privateNetwork = true;
-  config = mkMerge (guestCfg.modules
+  # We bind-mount stuff from the host into /guest first, and later bind
-    ++ [
+  # mount them into the correct path inside the guest, so we have a
-      (import ./common-guest-config.nix guestName guestCfg)
+  # fileSystems entry that impermanence can depend upon.
-      {
+  bindMounts = {
-        systemd.network.networks = {
+    "/guest/state" = {
-          "10-${guestCfg.networking.mainLinkName}" = {
+      hostPath = "/state/guests/${guestName}";
-            matchConfig.OriginalName = "mv-${guestCfg.container.macvlan}";
+      isReadOnly = false;
-            linkConfig.Name = guestCfg.networking.mainLinkName;
+    };
+    # Mount persistent data from the host
+    "/guest/persist" = lib.mkIf guestCfg.zfs.enable {
+      hostPath = guestCfg.zfs.mountpoint;
+      isReadOnly = false;
+    };
+  };
+  nixosConfiguration = inputs.nixpkgs.lib.nixosSystem {
+    specialArgs = {
+      inherit lib nodes inputs minimal;
+    };
+    prefix = ["nodes" "${config.node.name}-${guestName}" "config"];
+    system = null;
+    modules =
+      [
+        {
+          boot.isContainer = true;
+          networking.useHostResolvConf = false;
+
+          # We cannot force the package set via nixpkgs.pkgs and
+          # inputs.nixpkgs.nixosModules.readOnlyPkgs, since some nixosModules
+          # like nixseparatedebuginfod depend on adding packages via nixpkgs.overlays.
+          # So we just mimic the options and overlays defined by the passed pkgs set.
+          nixpkgs.hostPlatform = config.nixpkgs.hostPlatform.system;
+          nixpkgs.overlays = pkgs.overlays;
+          nixpkgs.config = pkgs.config;
+
+          # Bind the /guest/* paths from above so impermancence doesn't complain.
+          fileSystems."/state" = {
+            fsType = "none";
+            neededForBoot = true;
+            device = "/guest/state";
+            options = ["bind"];
           };
-        };
+          fileSystems."/persist" = lib.mkIf guestCfg.zfs.enable {
-      }
+            fsType = "none";
-    ]);
+            neededForBoot = true;
+            device = "/guest/persist";
+            options = ["bind"];
+          };
+        }
+        (import ./common-guest-config.nix guestName guestCfg)
+      ]
+      ++ guestCfg.modules;
+  };
 }

modules/guests/default.nix

@@ -103,6 +103,19 @@ in {
     default = "host";
   };
 
+  options.containers = mkOption {
+    type = types.attrsOf (types.submodule (submod: {
+      options.nixosConfiguration = mkOption {
+        type = types.unspecified;
+        default = null;
+        description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
+      };
+      config = mkIf (submod.config.nixosConfiguration != null) {
+        path = submod.config.nixosConfiguration.config.system.build.toplevel;
+      };
+    }));
+  };
+
   options.guests = mkOption {
     default = {};
     description = "Defines the actual vms and handles the necessary base setup for them.";

modules/guests/microvm.nix

@@ -18,7 +18,6 @@ guestName: guestCfg: {
   mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames config.guests)).${guestName};
 in {
   specialArgs = {
-    parentNode = config;
     inherit (inputs.self) nodes;
     inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib;
     inherit inputs;
@@ -77,7 +76,7 @@ in {
         };
     };
 
-    # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here
+    # FIXME this should be changed in microvm.nix to mkDefault in order to not require mkForce here
     fileSystems."/state".neededForBoot = mkForce true;
     fileSystems."/persist".neededForBoot = mkForce true;
 

nix/hosts.nix

@@ -52,16 +52,16 @@ inputs: let
   # together with it. We collect all defined guests from each node here
   # to allow accessing any node via the unified attribute `nodes`.
   guestConfigs = flip concatMapAttrs self.nixosConfigurations (_: node:
-    flip mapAttrs' (node.config.guests or {}) (guestName: guestDef:
+    flip mapAttrs' (node.config.guests or {}) (
-      nameValuePair guestDef.nodeName (
+      guestName: guestDef:
-        if guestDef.backend == "microvm"
+        nameValuePair guestDef.nodeName
-        then node.config.microvm.vms.${guestName}.config
+        (
-        else {
+          if guestDef.backend == "microvm"
-          # We can only access the .config part of nixosSystem here unfortunately,
+          then node.config.microvm.vms.${guestName}.config
-          # since the rest is not exposed by the nixos module.
+          else node.config.containers.${guestName}
-          inherit (node.config.containers.${guestName}) config;
+        )
-        }
+        .nixosConfiguration
-      )));
+    ));
 in {
   inherit
     hosts