feat: switch from colmena.makeHive to nixosSystem

STRUCTURE.md

@@ -35,10 +35,8 @@ Make sure to utilize the github search if you know what you need!
   - `modules/*/` regular modules related to <xyz>, similar structure as in `nixpkgs/nixos/modules`
 
 - `nix/` library functions and flake plumbing
-  - `colmena.nix` Setup for distributed deployment using colmena (actually defines all NixOS hosts)
-  - `extra-builtins.nix` Extra builtins via nix-plugins to support transparent repository-wide secrets
   - `generate-installer-package.nix` Helper package that that will be available in our iso images. This provides the `install-system` command that will do a full install including partitioning.
-  - `generate-node.nix` Helper function that outputs everything that is necessary to define a new node in a predictable format. Used to define colmena nodes and microvms.
+  - `hosts.nix` Loads all host declarations from host.toml and defines the actual hosts in nixosConfigurations.
   - `installer-configuration.nix` Our modified ISO installer image config (sets up ssh, contains the installer package, ...)
   - `rage-decrypt-and-cache.sh` Auxiliary script for repository-wide secrets that decrypts a file and caches the output in /tmp
 

flake.nix

@@ -1,5 +1,5 @@
 {
-  description = "oddlama's NixOS Infrastructure";
+  description = " ❄️ oddlama's nix config and dotfiles";
 
   inputs = {
     colmena = {
@@ -96,17 +96,20 @@
     agenix-rekey,
     colmena,
     devshell,
-    elewrap,
     flake-utils,
-    microvm,
     nixos-generators,
     nixpkgs,
-    nixpkgs-wayland,
-    nixseparatedebuginfod,
     pre-commit-hooks,
     ...
   } @ inputs: let
-    inherit (nixpkgs) lib;
+    inherit
+      (nixpkgs.lib)
+      cleanSource
+      foldl'
+      mapAttrs
+      mapAttrsToList
+      recursiveUpdate
+      ;
   in
     {
       # The identities that are used to rekey agenix secrets and to
@@ -116,39 +119,27 @@
         extraEncryptionPubkeys = [./secrets/backup.pub];
       };
 
-      # Load the list of hosts that this flake defines, which
+      inherit
-      # associates the minimum amount of metadata that is necessary
+        (import ./nix/hosts.nix inputs)
-      # to instanciate hosts correctly.
+        colmena
-      hosts = builtins.fromTOML (builtins.readFile ./hosts.toml);
+        hosts
-
+        microvmConfigurations
-      # This will process all defined hosts of type "nixos" and
+        nixosConfigurations
-      # generate the required colmena definition for each host.
+        ;
-      # We call the resulting instanciations "nodes".
-      # TODO: switch to nixosConfigurations once colmena supports it upstream
-      colmena = import ./nix/colmena.nix inputs;
-      colmenaNodes = ((colmena.lib.makeHive self.colmena).introspect (x: x)).nodes;
-
-      # True NixOS nodes can define additional microvms (guest nodes) that are built
-      # together with the true host. We collect all defined microvm nodes
-      # from each node here to allow accessing any node via the unified attribute `nodes`.
-      microvmNodes = lib.flip lib.concatMapAttrs self.colmenaNodes (_: node:
-        lib.mapAttrs'
-        (vm: def: lib.nameValuePair def.nodeName node.config.microvm.vms.${vm}.config)
-        (node.config.meta.microvms.vms or {}));
 
       # All nixosSystem instanciations are collected here, so that we can refer
       # to any system via nodes.<name>
-      nodes = self.colmenaNodes // self.microvmNodes;
+      nodes = self.nixosConfigurations // self.microvmConfigurations;
       # Add a shorthand to easily target toplevel derivations
-      "@" = lib.mapAttrs (_: v: v.config.system.build.toplevel) self.nodes;
+      "@" = mapAttrs (_: v: v.config.system.build.toplevel) self.nodes;
 
       # For each true NixOS system, we want to expose an installer package that
       # can be used to do the initial setup on the node from a live environment.
       inherit
-        (lib.foldl' lib.recursiveUpdate {}
+        (foldl' recursiveUpdate {}
-          (lib.mapAttrsToList
+          (mapAttrsToList
             (import ./nix/generate-installer-package.nix inputs)
-            self.colmenaNodes))
+            self.nixosConfigurations))
         packages
         ;
     }
@@ -161,10 +152,6 @@
           ++ import ./pkgs/default.nix
           ++ [
             devshell.overlays.default
-            elewrap.overlays.default
-            microvm.overlay
-            nixpkgs-wayland.overlay
-            nixseparatedebuginfod.overlays.default
           ];
       };
 
@@ -193,7 +180,7 @@
 
       # `nix flake check`
       checks.pre-commit-hooks = pre-commit-hooks.lib.${system}.run {
-        src = lib.cleanSource ./.;
+        src = cleanSource ./.;
         hooks = {
           # Nix
           alejandra.enable = true;
@@ -210,7 +197,7 @@
         name = "nix-config";
         packages = with pkgs; [
           faketty # Used in my colmena patch to show progress, XXX: should theoretically be propagated automatically from the patch....
-          nix # Always use the nix version from this flake's nixpkgs versios, so that nix-plugins (below) doesn't fail because of different nix versions.
+          nix # Always use the nix version from this flake's nixpkgs version, so that nix-plugins (below) doesn't fail because of different nix versions.
         ];
 
         commands = with pkgs; [

modules/default.nix

@@ -1,6 +1,15 @@
-{
+{inputs, ...}: {
   disabledModules = ["services/security/kanidm.nix"];
   imports = [
+    inputs.agenix-rekey.nixosModules.default
+    inputs.agenix.nixosModules.default
+    inputs.disko.nixosModules.disko
+    inputs.elewrap.nixosModules.default
+    inputs.home-manager.nixosModules.default
+    inputs.impermanence.nixosModules.impermanence
+    inputs.nixos-nftables-firewall.nixosModules.default
+    inputs.nixseparatedebuginfod.nixosModules.default
+
     ../users/root
 
     ./config/boot.nix
@@ -38,4 +47,9 @@
 
     ./system/deteministic-ids.nix
   ];
+
+  nixpkgs.overlays = [
+    inputs.microvm.overlay
+    inputs.nixpkgs-wayland.overlay
+  ];
 }

modules/meta/microvms.nix

@@ -84,23 +84,21 @@
       };
 
     microvm.vms.${vmName} = let
-      node = import ../../nix/generate-node.nix inputs {
-        name = vmCfg.nodeName;
-        inherit (vmCfg) system;
-      };
       mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames vms)).${vmName};
     in {
       # Allow children microvms to know which node is their parent
-      specialArgs =
+      specialArgs = {
-        {
         parentNode = config;
         parentNodeName = nodeName;
-        }
+        inherit (inputs.self) nodes;
-        // node.specialArgs;
+        inherit (inputs.self.pkgs.${vmCfg.system}) lib;
-      inherit (node) pkgs;
+        inherit inputs;
+      };
+      pkgs = inputs.self.pkgs.${vmCfg.system};
       inherit (vmCfg) autostart;
       config = {config, ...}: {
-        imports = cfg.commonImports ++ node.imports ++ vmCfg.modules;
+        imports = cfg.commonImports ++ vmCfg.modules;
+        node.name = vmCfg.nodeName;
 
         lib.microvm.mac = mac;
 

modules/optional/hardware/physical.nix

@@ -8,6 +8,6 @@
   services = {
     fwupd.enable = true;
     smartd.enable = true;
-    thermald.enable = builtins.elem config.nixpkgs.system ["x86_64-linux"];
+    thermald.enable = builtins.elem config.nixpkgs.hostPlatform.system ["x86_64-linux"];
   };
 }

nix/colmena.nix

@@ -1,33 +0,0 @@
-{
-  self,
-  nixpkgs,
-  ...
-} @ inputs: let
-  inherit
-    (nixpkgs.lib)
-    filterAttrs
-    flip
-    mapAttrs
-    ;
-
-  nixosNodes = filterAttrs (_: x: x.type == "nixos") self.hosts;
-  nodes = flip mapAttrs nixosNodes (name: hostCfg:
-    import ./generate-node.nix inputs {
-      inherit name;
-      inherit (hostCfg) system;
-      modules = [
-        ../hosts/${name}
-        {node.secretsDir = ../hosts/${name}/secrets;}
-      ];
-    });
-in
-  {
-    meta = {
-      description = "❄️";
-      # Just a required dummy for colmena, overwritten on a per-node basis by nodeNixpkgs below.
-      nixpkgs = self.pkgs.x86_64-linux;
-      nodeNixpkgs = mapAttrs (_: node: node.pkgs) nodes;
-      nodeSpecialArgs = mapAttrs (_: node: node.specialArgs) nodes;
-    };
-  }
-  // mapAttrs (_: node: {inherit (node) imports;}) nodes

nix/generate-node.nix

@@ -1,41 +0,0 @@
-{
-  self,
-  agenix,
-  agenix-rekey,
-  disko,
-  elewrap,
-  home-manager,
-  impermanence,
-  nixos-nftables-firewall,
-  nixseparatedebuginfod,
-  ...
-} @ inputs: {
-  # The name of the generated node
-  name,
-  # Additional modules that should be imported
-  modules ? [],
-  # The system in use
-  system,
-  ...
-}: {
-  inherit system;
-  pkgs = self.pkgs.${system};
-  specialArgs = {
-    inherit (self.pkgs.${system}) lib;
-    inherit (self) nodes;
-    inherit inputs;
-  };
-  imports =
-    modules
-    ++ [
-      {node.name = name;}
-      agenix-rekey.nixosModules.default
-      agenix.nixosModules.default
-      disko.nixosModules.disko
-      elewrap.nixosModules.default
-      home-manager.nixosModules.default
-      impermanence.nixosModules.impermanence
-      nixos-nftables-firewall.nixosModules.default
-      nixseparatedebuginfod.nixosModules.default
-    ];
-}

nix/hosts.nix

@@ -0,0 +1,80 @@
+inputs: let
+  inherit (inputs) self;
+  inherit
+    (inputs.nixpkgs.lib)
+    concatMapAttrs
+    filterAttrs
+    flip
+    mapAttrs
+    mapAttrs'
+    nameValuePair
+    nixosSystem
+    ;
+
+  mapNixosConfigs = f: mapAttrs (_: f) self.nixosConfigurations;
+
+  # Creates a new nixosSystem with the correct specialArgs, pkgs and name definition
+  mkHost = name: system: let
+    pkgs = self.pkgs.${system};
+  in
+    nixosSystem {
+      specialArgs = {
+        # Use the correct instance lib that has our overlays
+        inherit (pkgs) lib;
+        inherit (self) nodes;
+        inherit inputs;
+      };
+      modules = [
+        {
+          # We cannot force the package set via nixpkgs.pkgs and
+          # inputs.nixpkgs.nixosModules.readOnlyPkgs, since some nixosModules
+          # like nixseparatedebuginfod depend on adding packages via nixpkgs.overlays.
+          # So we just mimic the options and overlays defined by the passed pkgs set.
+          nixpkgs.hostPlatform = system;
+          nixpkgs.overlays = pkgs.overlays;
+          nixpkgs.config = pkgs.config;
+          node.name = name;
+          node.secretsDir = ../hosts/${name}/secrets;
+        }
+        ../hosts/${name}
+        inputs.colmena.nixosModules.assertionModule
+        inputs.colmena.nixosModules.deploymentOptions
+      ];
+    };
+
+  # Load the list of hosts that this flake defines, which
+  # associates the minimum amount of metadata that is necessary
+  # to instanciate hosts correctly.
+  hosts = builtins.fromTOML (builtins.readFile ../hosts.toml);
+  # Get all hosts of type "nixos"
+  nixosHosts = filterAttrs (_: x: x.type == "nixos") hosts;
+  # Process each nixosHosts declaration and generatea nixosSystem definitions
+  nixosConfigurations = flip mapAttrs nixosHosts (name: hostCfg: mkHost name hostCfg.system);
+
+  # We now wrap nixosConfigurations so that colmena understands it
+  colmena =
+    {
+      meta = {
+        # Just a required dummy for colmena, overwritten on a per-node basis by nodeNixpkgs below.
+        nixpkgs = self.pkgs.x86_64-linux;
+        nodeNixpkgs = mapNixosConfigs (v: v.pkgs);
+        nodeSpecialArgs = mapNixosConfigs (v: v._module.specialArgs);
+      };
+    }
+    // mapNixosConfigs (v: {imports = v._module.args.modules;});
+
+  # True NixOS nodes can define additional microvms (guest nodes) that are built
+  # together with the true host. We collect all defined microvm nodes
+  # from each node here to allow accessing any node via the unified attribute `nodes`.
+  microvmConfigurations = flip concatMapAttrs self.nixosConfigurations (_: node:
+    mapAttrs'
+    (vm: def: nameValuePair def.nodeName node.config.microvm.vms.${vm}.config)
+    (node.config.meta.microvms.vms or {}));
+in {
+  inherit
+    colmena
+    hosts
+    microvmConfigurations
+    nixosConfigurations
+    ;
+}