refactor: properly modularize repo secret management

flake.nix

@@ -82,10 +82,9 @@
 
       # The identities that are used to rekey agenix secrets and to
       # decrypt all repository-wide secrets.
-      secrets = {
+      secretsConfig = {
         masterIdentities = [./secrets/yk1-nix-rage.pub];
         extraEncryptionPubkeys = [./secrets/backup.pub];
-        content = import ./nix/secrets.nix inputs;
       };
 
       stateVersion = "23.05";
@@ -112,6 +111,7 @@
         (nodeName: nodeAttrs:
           nixpkgs.lib.mapAttrs'
           # TODO This is duplicated three times. This is microvm naming #3
+          # TODO maybe use microvm.vms.<name>.compoundName
           (n: nixpkgs.lib.nameValuePair "${nodeName}-${n}")
           (self.colmenaNodes.${nodeName}.config.microvm.vms or {}))
         self.colmenaNodes;

hosts/common/core/default.nix

@@ -13,6 +13,7 @@
     ../../../modules/interface-naming.nix
     ../../../modules/microvms.nix
     ../../../modules/wireguard.nix
+    ../../../modules/repo.nix
   ];
 
   home-manager = {

hosts/common/core/net.nix

@@ -3,7 +3,6 @@
   lib,
   pkgs,
   nodeName,
-  nodeSecrets,
   ...
 }: let
   inherit
@@ -78,5 +77,5 @@ in {
   systemd.network.enable = true;
 
   # Rename known network interfaces
-  extra.networking.renameInterfacesByMac = lib.mapAttrs (_: v: v.mac) (nodeSecrets.networking.interfaces or {});
+  extra.networking.renameInterfacesByMac = lib.mapAttrs (_: v: v.mac) (config.repo.secrets.local.networking.interfaces or {});
 }

hosts/common/core/system.nix

@@ -177,10 +177,19 @@
       };
     };
 
+  # Define local repo secrets
+  repo.secretFiles = let
+    local = nodePath + "/secrets/local.nix.age";
+  in
+    {
+      global = ../../../secrets/global.nix.age;
+    }
+    // lib.optionalAttrs (nodePath != null && lib.pathExists local) {inherit local;};
+
   # Setup secret rekeying parameters
   rekey = {
     inherit
-      (inputs.self.secrets)
+      (inputs.self.secretsConfig)
       masterIdentities
       extraEncryptionPubkeys
       ;

hosts/common/dev/default.nix

@@ -1,8 +1,8 @@
 {
   imports = [
     ./documentation.nix
-    ./nix.nix
   ];
 
   environment.enableDebugInfo = true;
+  repo.defineNixExtraBuiltins = true;
 }

hosts/common/dev/nix.nix

@@ -1,8 +0,0 @@
-{pkgs, ...}: {
-  # Make sure not to reference the extra-builtins file directly but
-  # at least via its parent folder so it can access relative files.
-  nix.extraOptions = ''
-    plugin-files = ${pkgs.nix-plugins}/lib/nix/plugins
-    extra-builtins-file = ${../../../nix}/extra-builtins.nix
-  '';
-}

hosts/nom/net.nix

@@ -1,10 +1,6 @@
-{
-  config,
-  nodeSecrets,
-  ...
-}: {
+{config, ...}: {
   networking = {
-    inherit (nodeSecrets.networking) hostId;
+    inherit (config.repo.secrets.local.networking) hostId;
     wireless.iwd.enable = true;
   };
 
@@ -16,14 +12,14 @@
   systemd.network.networks = {
     "10-lan1" = {
       DHCP = "yes";
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.lan1.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.lan1.mac;
       networkConfig.IPv6PrivacyExtensions = "yes";
       dhcpV4Config.RouteMetric = 10;
       dhcpV6Config.RouteMetric = 10;
     };
     "10-wlan1" = {
       DHCP = "yes";
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.wlan1.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.wlan1.mac;
       networkConfig.IPv6PrivacyExtensions = "yes";
       dhcpV4Config.RouteMetric = 40;
       dhcpV6Config.RouteMetric = 40;

hosts/ward/fs.nix

@@ -1,7 +1,7 @@
 {
+  config,
   lib,
   extraLib,
-  nodeSecrets,
   pkgs,
   ...
 }: {
@@ -9,7 +9,7 @@
     disk = {
       m2-ssd = {
         type = "disk";
-        device = "/dev/disk/by-id/${nodeSecrets.disk.m2-ssd}";
+        device = "/dev/disk/by-id/${config.repo.secrets.local.disk.m2-ssd}";
         content = with extraLib.disko.gpt; {
           type = "table";
           format = "gpt";

hosts/ward/net.nix

@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  nodeSecrets,
   ...
 }: let
   inherit (config.lib.net) ip cidr;
@@ -9,7 +8,7 @@
   lanCidrv4 = "192.168.100.0/24";
   lanCidrv6 = "fd00::/64";
 in {
-  networking.hostId = nodeSecrets.networking.hostId;
+  networking.hostId = config.repo.secrets.local.networking.hostId;
 
   boot.initrd.systemd.network = {
     enable = true;
@@ -31,7 +30,7 @@ in {
 
   systemd.network.networks = {
     "10-lan" = {
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.lan.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.lan.mac;
       # This interface should only be used from attached macvtaps.
       # So don't acquire a link local address and only wait for
       # this interface to gain a carrier.
@@ -50,7 +49,7 @@ in {
       #];
       #gateway = [
       #];
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.wan.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.wan.mac;
       networkConfig.IPv6PrivacyExtensions = "yes";
       linkConfig.RequiredForOnline = "routable";
     };
@@ -183,7 +182,7 @@ in {
   systemd.services.kea-dhcp4-server.after = ["sys-subsystem-net-devices-lan.device"];
 
   extra.microvms.networking = {
-    baseMac = nodeSecrets.networking.interfaces.lan.mac;
+    baseMac = config.repo.secrets.local.networking.interfaces.lan.mac;
     macvtapInterface = "lan";
     static = {
       baseCidrv4 = lanCidrv4;

hosts/ward/vaultwarden.nix

@@ -1,8 +1,4 @@
-{
-  config,
-  nodeSecrets,
-  ...
-}: {
+{config, ...}: {
   services.vaultwarden = {
     enable = true;
     dbBackend = "sqlite";
@@ -22,7 +18,7 @@
       PASSWORD_ITERATIONS = 1000000;
       INVITATIONS_ALLOWED = true;
       INVITATION_ORG_NAME = "Vaultwarden";
-      DOMAIN = nodeSecrets.vaultwarden.domain;
+      DOMAIN = config.repo.secrets.local.vaultwarden.domain;
 
       SMTP_EMBED_IMAGES = true;
     };
@@ -59,7 +55,7 @@
         keepalive 2;
       '';
     };
-    virtualHosts."${nodeSecrets.vaultwarden.domain}" = {
+    virtualHosts."${config.repo.secrets.local.vaultwarden.domain}" = {
       forceSSL = true;
       #enableACME = true;
       sslCertificate = config.rekey.secrets."selfcert.crt".path;

hosts/zackbiene/esphome.nix

@@ -1,7 +1,6 @@
 {
   lib,
   config,
-  nodeSecrets,
   ...
 }: {
   services.esphome = {
@@ -24,7 +23,7 @@
         keepalive 2;
       '';
     };
-    virtualHosts."${nodeSecrets.esphome.domain}" = {
+    virtualHosts."${config.repo.secrets.local.esphome.domain}" = {
       forceSSL = true;
       #enableACME = true;
       sslCertificate = config.rekey.secrets."selfcert.crt".path;

hosts/zackbiene/home-assistant.nix

@@ -1,7 +1,6 @@
 {
   lib,
   config,
-  nodeSecrets,
   ...
 }: let
   haPort = 8123;
@@ -115,7 +114,7 @@ in {
         keepalive 2;
       '';
     };
-    virtualHosts."${nodeSecrets.homeassistant.domain}" = {
+    virtualHosts."${config.repo.secrets.local.homeassistant.domain}" = {
       serverAliases = ["192.168.1.21"]; # TODO remove later
       forceSSL = true;
       #enableACME = true;

hosts/zackbiene/hostapd.nix

@@ -2,7 +2,6 @@
   lib,
   config,
   pkgs,
-  nodeSecrets,
   ...
 }: {
   imports = [../../modules/hostapd.nix];
@@ -19,7 +18,7 @@
       channel = 13; # Automatic Channel Selection (ACS) is unfortunately not implemented for mt7612u.
       wifi4.capabilities = ["LDPC" "HT40+" "HT40-" "GF" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1"];
       networks.wlan1 = {
-        inherit (nodeSecrets.hostapd) ssid;
+        inherit (config.repo.secrets.local.hostapd) ssid;
         macAcl = "allow";
         apIsolate = true;
         authentication = {
@@ -30,7 +29,7 @@
         bssid = "00:c0:ca:b1:4f:9f";
       };
       #networks.wlan1-2 = {
-      #  inherit (nodeSecrets.hostapd) ssid;
+      #  inherit (config.repo.secrets.local.hostapd) ssid;
       #  authentication.mode = "none";
       #  bssid = "02:c0:ca:b1:4f:9f";
       #};

hosts/zackbiene/net.nix

@@ -1,7 +1,6 @@
 {
   lib,
   config,
-  nodeSecrets,
   ...
 }: let
   inherit (config.lib.net) cidr;
@@ -9,7 +8,7 @@
   net.iot.ipv4cidr = "10.90.0.1/24";
   net.iot.ipv6cidr = "fd90::1/64";
 in {
-  networking.hostId = nodeSecrets.networking.hostId;
+  networking.hostId = config.repo.secrets.local.networking.hostId;
 
   boot.initrd.systemd.network = {
     enable = true;
@@ -19,13 +18,13 @@ in {
   systemd.network.networks = {
     "10-lan1" = {
       DHCP = "yes";
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.lan1.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.lan1.mac;
       networkConfig.IPv6PrivacyExtensions = "yes";
       linkConfig.RequiredForOnline = "routable";
     };
     "10-wlan1" = {
       address = [net.iot.ipv4cidr net.iot.ipv6cidr];
-      matchConfig.MACAddress = nodeSecrets.networking.interfaces.wlan1.mac;
+      matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.wlan1.mac;
       linkConfig.RequiredForOnline = "no";
     };
   };

hosts/zackbiene/nginx.nix

@@ -1,7 +1,6 @@
 {
   lib,
   config,
-  nodeSecrets,
   ...
 }: {
   rekey.secrets."selfcert.crt" = {

hosts/zackbiene/zigbee2mqtt.nix

@@ -1,7 +1,6 @@
 {
   lib,
   config,
-  nodeSecrets,
   ...
 }: {
   rekey.secrets."mosquitto-pw-zigbee2mqtt.yaml" = {
@@ -39,7 +38,7 @@
         keepalive 2;
       '';
     };
-    virtualHosts."${nodeSecrets.zigbee2mqtt.domain}" = {
+    virtualHosts."${config.repo.secrets.local.zigbee2mqtt.domain}" = {
       forceSSL = true;
       #enableACME = true;
       sslCertificate = config.rekey.secrets."selfcert.crt".path;

modules/repo.nix

@@ -0,0 +1,93 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  ...
+}: let
+  inherit
+    (lib)
+    assertMsg
+    attrNames
+    literalExpression
+    mapAttrs
+    mkIf
+    mkOption
+    types
+    ;
+
+  # If the given expression is a bare set, it will be wrapped in a function,
+  # so that the imported file can always be applied to the inputs, similar to
+  # how modules can be functions or sets.
+  constSet = x:
+    if builtins.isAttrs x
+    then (_: x)
+    else x;
+
+  # Try to access the extra builtin we loaded via nix-plugins.
+  # Throw an error if that doesn't exist.
+  rageImportEncrypted = assert assertMsg (builtins ? extraBuiltins.rageImportEncrypted) "The extra builtin 'rageImportEncrypted' is not available, so repo.secrets cannot be decrypted. Did you forget to use `defineNixExtraBuiltins` or use the appropriate ad-hoc command line arguments?";
+    builtins.extraBuiltins.rageImportEncrypted;
+
+  # This "imports" an encrypted .nix.age file by evaluating the decrypted content.
+  importEncrypted = path:
+    constSet (
+      if builtins.pathExists path
+      then rageImportEncrypted inputs.self.secretsConfig.masterIdentities path
+      else {}
+    );
+
+  cfg = config.repo;
+in {
+  options.repo = {
+    defineNixExtraBuiltins = mkOption {
+      default = false;
+      type = types.bool;
+      description = ''
+        Add nix-plugins and the correct extra-builtin-files definition to this host's
+        nix configuration, so that it can be used to decrypt the secrets in this repository.
+      '';
+    };
+
+    secretFiles = mkOption {
+      default = {};
+      type = types.attrsOf types.path;
+      example = literalExpression "{ local = ./secrets.nix.age; }";
+      description = ''
+        This file manages the origin for this machine's repository-secrets. Anything that is
+        technically not a secret in the classical sense (i.e. that it has to be protected
+        after it has been deployed), but something you want to keep secret from the public;
+        Anything that you wouldn't want people to see on GitHub, but that can live unencrypted
+        on your own devices. Consider it a more ergonomic nix alternative to using git-crypt.
+
+        All of these secrets may (and probably will be) put into the world-readable nix-store
+        on the build and target hosts. You'll most likely want to store personally identifiable
+        information here, such as:
+          - MAC Addreses
+          - Static IP addresses
+          - Your full name (when configuring your users)
+          - Your postal address (when configuring e.g. home-assistant)
+          - ...
+
+        Each path given here must be an age-encrypted .nix file. For each attribute `<name>`,
+        the corresponding file will be decrypted, imported and exposed as {option}`repo.secrets.<name>`.
+      '';
+    };
+
+    secrets = mkOption {
+      readOnly = true;
+      default = mapAttrs (_: x: importEncrypted x inputs) cfg.secretFiles;
+      type = types.unspecified;
+      description = "Exposes the loaded repo secrets. This option is read-only.";
+    };
+  };
+
+  config = {
+    # Make sure not to reference the extra-builtins file directly but
+    # at least via its parent folder so it can access relative files.
+    nix.extraOptions = mkIf cfg.defineNixExtraBuiltins ''
+      plugin-files = ${pkgs.nix-plugins}/lib/nix/plugins
+      extra-builtins-file = ${../nix}/extra-builtins.nix
+    '';
+  };
+}

nix/colmena.nix

@@ -10,10 +10,10 @@
     ;
 
   nixosNodes = filterAttrs (_: x: x.type == "nixos") self.hosts;
-  nodes = mapAttrs (import ./generate-node.nix inputs) nixosNodes;
-  generateColmenaNode = nodeName: _: {
-    inherit (nodes.${nodeName}) imports;
-  };
+  nodes =
+    mapAttrs
+    (n: v: import ./generate-node.nix inputs n ({config = ../hosts/${n};} // v))
+    nixosNodes;
 in
   {
     meta = {
@@ -24,4 +24,4 @@ in
       nodeSpecialArgs = mapAttrs (_: node: node.specialArgs) nodes;
     };
   }
-  // mapAttrs generateColmenaNode nodes
+  // mapAttrs (_: node: {inherit (node) imports;}) nodes

nix/generate-node.nix

@@ -13,23 +13,24 @@
   ...
 } @ inputs: let
   inherit (nixpkgs.lib) optionals;
+  pathOrNull = x:
+    if builtins.isPath x
+    then x
+    else null;
 in
-  nodeName: nodeMeta: let
-    nodePath = nodeMeta.config or (../hosts + "/${nodeName}");
-  in {
+  nodeName: nodeMeta: {
     inherit (nodeMeta) system;
     pkgs = self.pkgs.${nodeMeta.system};
     specialArgs = {
       inherit (nixpkgs) lib;
       inherit (self) extraLib nodes stateVersion;
-      inherit inputs nodeName nodePath;
-      secrets = self.secrets.content;
-      nodeSecrets = self.secrets.content.nodes.${nodeName} or {};
+      inherit inputs nodeName;
+      nodePath = pathOrNull (nodeMeta.config or null);
       nixos-hardware = nixos-hardware.nixosModules;
       microvm = microvm.nixosModules;
     };
     imports = [
-      nodePath # default module
+      (nodeMeta.config or {})
       agenix.nixosModules.default
       agenix-rekey.nixosModules.default
       disko.nixosModules.disko

nix/secrets.nix

@@ -1,65 +0,0 @@
-# This file manages access to repository-secrets. Anything that is technically
-# not a secret on your hosts, but something you want to keep secret from the public.
-# Anything you don't want people to see on GitHub that isn't a password or encrypted
-# using agenix.
-#
-# All of these secrets may (and probably will be) put into the world-readable nix-store
-# on the build and target hosts. You'll most likely want to store personally identifiable
-# information here, such as:
-#   - MAC Addreses
-#   - Static IP addresses
-#   - Your full name (when configuring e.g. users)
-#   - Your postal address (when configuring e.g. home-assistant)
-#   - ...
-{
-  self,
-  nixpkgs,
-  ...
-} @ inputs: let
-  inherit
-    (nixpkgs.lib)
-    attrNames
-    concatMap
-    filterAttrs
-    listToAttrs
-    mapAttrs
-    nameValuePair
-    ;
-  # If the given expression is a bare set, it will be wrapped in a function,
-  # so that the imported file can always be applied to the inputs, similar to
-  # how modules can be functions or sets.
-  constSet = x:
-    if builtins.isAttrs x
-    then (_: x)
-    else x;
-  # This "imports" an encrypted .nix.age file
-  importEncrypted = path:
-    constSet (
-      if builtins.pathExists path
-      then builtins.extraBuiltins.rageImportEncrypted self.secrets.masterIdentities path
-      else {}
-    );
-
-  # Secrets for each physical node
-  nodeSecrets = mapAttrs (nodeName: _: importEncrypted ../hosts/${nodeName}/secrets/secrets.nix.age inputs) self.hosts;
-
-  # A list of all nodes that have microvm directories
-  nodesWithMicrovms = builtins.filter (nodeName: builtins.pathExists ../hosts/${nodeName}/microvms) (attrNames self.hosts);
-  # Returns a list of all microvms defined for the given node
-  microvmsFor = nodeName:
-    attrNames (filterAttrs
-      (_: t: t == "directory")
-      (builtins.readDir ../hosts/${nodeName}/microvms));
-  # Returns all defined microvms with name and definition for a given node
-  microvmDefsFor = nodeName:
-    map
-    # TODO This is duplicated three times. This is microvm naming #2
-    (microvmName: nameValuePair "${nodeName}-${microvmName}" ../hosts/${nodeName}/microvms/${microvmName})
-    (microvmsFor nodeName);
-  # A attrset mapping all microvm nodes to its definition folder
-  microvms = listToAttrs (concatMap microvmDefsFor nodesWithMicrovms);
-  # The secrets for each microvm
-  microvmSecrets = mapAttrs (microvmName: microvmPath: importEncrypted (microvmPath + "/secrets/secrets.nix.age") inputs) microvms;
-in
-  (importEncrypted ../secrets/secrets.nix.age inputs)
-  // {nodes = nodeSecrets // microvmSecrets;}

users/myuser/default.nix

@@ -2,11 +2,10 @@
   config,
   lib,
   pkgs,
-  secrets,
   stateVersion,
   ...
 }: let
-  inherit (secrets) myuser;
+  inherit (config.repo.secrets.global) myuser;
 in {
   users.groups.${myuser}.gid = config.users.users.${myuser}.uid;
   users.users.${myuser} = {