feat: automatically generate allowedTCPPorts for mdns enabled

interfaces; simplify nftables rules by adding a general untrusted zone

README.md

@@ -117,3 +117,31 @@ openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
   -keyout selfcert.key -out selfcert.crt -subj \
   "/CN=example.com" -addext "subjectAltName=DNS:example.com,DNS:sub1.example.com,DNS:sub2.example.com,IP:10.0.0.1"
 ```
+
+
+
+
+
+
+
+
+
+
+```bash
+# Recover admin account (server must not be running)
+> systemctl stop kanidmd
+> kanidmd recover_account -c server.toml admin
+qU6UUdN5PbaetgtjKDttQx6D7XQwa0bBef5N5N0sjchg8gNz
+> systemctl start kanidmd
+# Login with recovered root account
+> kanidm login -D admin
+# Generate new credentials for idm_admin account
+> kanidm service-account credential generate -D admin idm_admin
+xbwa3tbUefdRBxKqbDYQfW2StqjZYa0zwp6FQRyWXy0dCYUb
+
+
+```
+
+
+
+

flake.nix

@@ -89,19 +89,16 @@
 
       stateVersion = "23.05";
 
-      hosts = {
+      hosts = let
-        nom = {
+        nixos = system: {
           type = "nixos";
-          system = "x86_64-linux";
+          inherit system;
-        };
-        ward = {
-          type = "nixos";
-          system = "x86_64-linux";
-        };
-        zackbiene = {
-          type = "nixos";
-          system = "aarch64-linux";
         };
+      in {
+        nom = nixos "x86_64-linux";
+        #sentinel = nixos "x86_64-linux";
+        ward = nixos "x86_64-linux";
+        zackbiene = nixos "aarch64-linux";
       };
 
       colmena = import ./nix/colmena.nix inputs;

hosts/common/core/default.nix

@@ -12,6 +12,7 @@
 
     ../../../users/root
 
+    ../../../modules/extra.nix
     ../../../modules/interface-naming.nix
     ../../../modules/microvms.nix
     ../../../modules/wireguard.nix

hosts/common/core/impermanence.nix

@@ -5,6 +5,8 @@
 }: {
   # State that should be kept across reboots, but is otherwise
   # NOT important information in any way that needs to be backed up.
+  #environment.persistence."/local" = {
+  # with new dataset   --> ^-- , or without v--
   #environment.persistence."/nix/state" = {
   #  hideMounts = true;
   #  files = [
@@ -32,12 +34,14 @@
           group = "root";
           mode = "0755";
         }
+        # TODO only persist across reboots, don't backup, once loki is used
         {
           directory = "/var/lib/systemd";
           user = "root";
           group = "root";
           mode = "0755";
         }
+        # TODO only persist across reboots, don't backup, once loki is used
         {
           directory = "/var/log";
           user = "root";
@@ -46,6 +50,7 @@
         }
         #{ directory = "/tmp"; user = "root"; group = "root"; mode = "1777"; }
         #{ directory = "/var/tmp"; user = "root"; group = "root"; mode = "1777"; }
+        # TODO only persist across reboots, don't backup, once loki is used
         {
           directory = "/var/spool";
           user = "root";

hosts/common/core/net.nix

@@ -70,6 +70,17 @@ in {
           to = ["local"];
           allowedTCPPorts = config.services.openssh.ports;
         };
+
+        untrusted-to-local = {
+          from = ["untrusted"];
+          to = ["local"];
+
+          inherit
+            (config.networking.firewall)
+            allowedTCPPorts
+            allowedUDPPorts
+            ;
+        };
       };
     };
   };

hosts/common/core/resolved.nix

@@ -1,9 +1,8 @@
-{lib, ...}: {
+{
-  networking.firewall = {
+  config,
-    allowedTCPPorts = [5355];
+  lib,
-    allowedUDPPorts = [5353 5355];
+  ...
-  };
+}: {
-
   services.resolved = {
     enable = true;
     dnssec = "allow-downgrade";
@@ -24,4 +23,53 @@
     (lib.mkBefore ["mdns_minimal [NOTFOUND=return]"])
     (lib.mkAfter ["mdns"])
   ];
+
+  # TODO mkForce nftables
+  # Open port 5353 for any interfaces that have MulticastDNS enabled
+  networking.nftables.firewall = let
+    # Determine all networks that have MulticastDNS enabled
+    networksWithMulticast =
+      lib.filter
+      (n: config.systemd.network.networks.${n}.networkConfig.MulticastDNS or false)
+      (lib.attrNames config.systemd.network.networks);
+
+    # Determine all known mac addresses and the corresponding link name
+    # based on the renameInterfacesByMac option.
+    knownMacs =
+      lib.mapAttrs'
+      (k: v: lib.nameValuePair v k)
+      config.extra.networking.renameInterfacesByMac;
+    # A helper that returns the link name for the given mac address,
+    # or null if it doesn't exist or the given mac was null.
+    linkNameFor = mac:
+      if mac == null
+      then null
+      else knownMacs.${mac} or null;
+
+    # Calls the given function for each network that has MulticastDNS enabled,
+    # and collects all non-null values.
+    mapNetworks = f: lib.filter (v: v != null) (map f networksWithMulticast);
+
+    # All interfaces on which MulticastDNS is used
+    mdnsInterfaces = lib.unique (
+      # For each network that is matched by MAC, lookup the link name
+      # and if map the definition name to the link name.
+      mapNetworks (x: linkNameFor (config.systemd.network.networks.${x}.matchConfig.MACAddress or null))
+      # For each network that is matched by name, map the definition
+      # name to the link name.
+      ++ mapNetworks (x: config.systemd.network.networks.${x}.matchConfig.Name or null)
+    );
+  in {
+    zones = lib.mkForce {
+      mdns.interfaces = mdnsInterfaces;
+    };
+
+    rules = lib.mkForce {
+      mdns-to-local = {
+        from = ["mdns"];
+        to = ["local"];
+        allowedUDPPorts = [5353];
+      };
+    };
+  };
 }

hosts/nom/net.nix

@@ -1,4 +1,8 @@
-{config, ...}: {
+{
+  config,
+  lib,
+  ...
+}: {
   networking = {
     inherit (config.repo.secrets.local.networking) hostId;
     wireless.iwd.enable = true;
@@ -31,4 +35,10 @@
       dhcpV6Config.RouteMetric = 40;
     };
   };
+
+  networking.nftables.firewall = {
+    zones = lib.mkForce {
+      untrusted.interfaces = ["lan1" "wlan1"];
+    };
+  };
 }

hosts/ward/default.nix

@@ -65,10 +65,6 @@ in {
 
     #automatic1111 = defineVm 19;
     #invokeai = defineVm 19;
-
-    #kanidm = defineVm 12 // {
-    #  configPath = ./vm-test.nix;
-    #};
   };
 
   microvm.vms.test.config = {
@@ -95,61 +91,22 @@ in {
       mode = "440";
       group = "acme";
     };
+
     security.acme = {
       acceptTerms = true;
       defaults = {
         inherit (acme) email;
-        dnsProvider = "cloudflare";
         credentialsFile = config.rekey.secrets.acme-credentials.path;
+        dnsProvider = "cloudflare";
         dnsPropagationCheck = true;
+        reloadServices = ["nginx"];
       };
-      certs = lib.genAttrs acme.domains (domain: {
-        extraDomainNames = ["*.${domain}"];
-      });
     };
+    extra.acme.wildcardDomains = acme.domains;
     users.groups.acme.members = ["nginx"];
+    services.nginx.enable = true;
 
-    # TODO reload nginx when acme is renewed
+    services.nginx = {
-
-    # TODO make default nginx config in core to reduce boilerplate?
-    services.nginx = let
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO not implemented well
-      # TODO (acme.domains is very specific)
-      # TODO (security.acme causes recursion)
-      matchingWildcardCert = domain: let
-        # Filter all certs that are wildcard certs and which match the given domain
-        matchingCerts =
-          lib.filter
-          (x: !lib.hasInfix "." (lib.removeSuffix ".${x}" domain))
-          acme.domains;
-      in
-        assert lib.assertMsg (matchingCerts != []) "No wildcard certificate was defined that matches ${domain}";
-          lib.head matchingCerts;
-    in {
-      enable = true;
-
-      recommendedBrotliSettings = true;
-      recommendedGzipSettings = true;
-      recommendedOptimisation = true;
-      recommendedProxySettings = true;
-      recommendedTlsSettings = true;
-
-      # SSL config
-      sslCiphers = "EECDH+AESGCM:EDH+AESGCM:!aNULL";
-      sslDhparam = config.rekey.secrets."dhparams.pem".path;
-      commonHttpConfig = ''
-        error_log syslog:server=unix:/dev/log;
-        access_log syslog:server=unix:/dev/log;
-        ssl_ecdh_curve secp384r1;
-      '';
-
       upstreams."kanidm" = {
         servers."${config.extra.wireguard."${parentNodeName}-local-vms".ipv4}:8300" = {};
         extraConfig = ''
@@ -159,7 +116,7 @@ in {
       };
       virtualHosts.${auth.domain} = {
         forceSSL = true;
-        useACMEHost = matchingWildcardCert auth.domain;
+        useACMEHost = config.lib.matchingWildcardCert auth.domain;
         locations."/".proxyPass = "https://kanidm";
         # Allow using self-signed certs to satisfy kanidm's requirement
         # for TLS connections. (This is over wireguard anyway)
@@ -169,10 +126,18 @@ in {
       };
     };
 
-    networking.firewall.allowedTCPPorts = [80 443];
+    networking.nftables.firewall = {
+      zones = lib.mkForce {
+        local-vms.interfaces = ["local-vms"];
+      };
 
-    networking.nftables.firewall.rules = lib.mkForce {
+      rules = lib.mkForce {
-      local-vms-to-local.allowedTCPPorts = [8300];
+        local-vms-to-local = {
+          from = ["local-vms"];
+          to = ["local"];
+          allowedTCPPorts = [8300];
+        };
+      };
     };
 
     rekey.secrets."kanidm-self-signed.crt" = {

hosts/ward/net.nix

@@ -89,9 +89,8 @@ in {
   # TODO mkForce nftables
   networking.nftables.firewall = {
     zones = lib.mkForce {
+      untrusted.interfaces = ["wan"];
       lan.interfaces = ["lan-self"];
-      wan.interfaces = ["wan"];
-      local-vms.interfaces = ["local-vms"];
     };
 
     rules = lib.mkForce {
@@ -100,34 +99,24 @@ in {
         extraLines = ["ip6 nexthdr icmpv6 icmpv6 type { mld-listener-query, nd-router-solicit } accept"];
       };
 
-      masquerade-wan = {
+      masquerade = {
         from = ["lan"];
-        to = ["wan"];
+        to = ["untrusted"];
         masquerade = true;
       };
 
+      # Rule needed to allow local-vms wireguard traffic
+      lan-to-local = {
+        from = ["lan"];
+        to = ["local"];
+      };
+
       outbound = {
         from = ["lan"];
-        to = ["lan" "wan"];
+        to = ["lan" "untrusted"];
         late = true; # Only accept after any rejects have been processed
         verdict = "accept";
       };
-
-      wan-to-local = {
-        from = ["wan"];
-        to = ["local"];
-      };
-
-      lan-to-local = {
-        from = ["lan"];
-        to = ["local"];
-
-        inherit
-          (config.networking.firewall)
-          allowedTCPPorts
-          allowedUDPPorts
-          ;
-      };
     };
   };
 

hosts/zackbiene/net.nix

@@ -38,20 +38,7 @@ in {
   # TODO mkForce nftables
   networking.nftables.firewall = {
     zones = lib.mkForce {
-      lan.interfaces = ["lan1"];
+      untrusted.interfaces = ["lan1"];
-    };
-
-    rules = lib.mkForce {
-      int-to-local = {
-        from = ["lan"];
-        to = ["local"];
-
-        inherit
-          (config.networking.firewall)
-          allowedTCPPorts
-          allowedUDPPorts
-          ;
-      };
     };
   };
 }

hosts/zackbiene/nginx.nix

@@ -21,24 +21,5 @@
 
   #security.acme.acceptTerms = true;
   #security.acme.defaults.email = "admin+acme@example.com";
-  services.nginx = {
+  services.nginx.enable = true;
-    enable = true;
-
-    recommendedBrotliSettings = true;
-    recommendedGzipSettings = true;
-    recommendedOptimisation = true;
-    recommendedProxySettings = true;
-    recommendedTlsSettings = true;
-
-    # SSL config
-    sslCiphers = "EECDH+AESGCM:EDH+AESGCM:!aNULL";
-    sslDhparam = config.rekey.secrets."dhparams.pem".path;
-    commonHttpConfig = ''
-      error_log syslog:server=unix:/dev/log;
-      access_log syslog:server=unix:/dev/log;
-      ssl_ecdh_curve secp384r1;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [80 443];
 }

modules/extra.nix

@@ -0,0 +1,71 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit
+    (lib)
+    assertMsg
+    filter
+    hasInfix
+    head
+    mdDoc
+    mkIf
+    mkOption
+    optionals
+    removeSuffix
+    types
+    ;
+in {
+  options.extra.acme.wildcardDomains = mkOption {
+    default = [];
+    example = ["example.org"];
+    type = types.listOf types.str;
+    description = mdDoc ''
+      All domains for which a wildcard certificate will be generated.
+      This will define the given `security.acme.certs` and set `extraDomainNames` correctly,
+      but does not fill any options such as credentials or dnsProvider. These have to be set
+      individually for each cert by the user or via `security.acme.defaults`.
+    '';
+  };
+
+  config = {
+    lib = {
+      # For a given domain, this searches for a matching wildcard acme domain that
+      # would include the given domain. If no such domain is defined in
+      # extra.acme.wildcardDomains, an assertion is triggered.
+      matchingWildcardCert = domain: let
+        matchingCerts =
+          filter
+          (x: !hasInfix "." (removeSuffix ".${x}" domain))
+          config.extra.acme.wildcardDomains;
+      in
+        assert assertMsg (matchingCerts != []) "No wildcard certificate was defined that matches ${domain}";
+          head matchingCerts;
+    };
+
+    security.acme.certs = lib.genAttrs config.extra.acme.wildcardDomains (domain: {
+      extraDomainNames = ["*.${domain}"];
+    });
+
+    # Sensible defaults for nginx
+    services.nginx = mkIf config.services.nginx.enable {
+      recommendedBrotliSettings = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedProxySettings = true;
+      recommendedTlsSettings = true;
+
+      # SSL config
+      sslCiphers = "EECDH+AESGCM:EDH+AESGCM:!aNULL";
+      sslDhparam = config.rekey.secrets."dhparams.pem".path;
+      commonHttpConfig = ''
+        error_log syslog:server=unix:/dev/log;
+        access_log syslog:server=unix:/dev/log;
+        ssl_ecdh_curve secp384r1;
+      '';
+    };
+
+    networking.firewall.allowedTCPPorts = optionals config.services.nginx.enable [80 443];
+  };
+}

modules/microvms.nix

@@ -75,7 +75,10 @@
           mkIf vmCfg.zfs.enable {
             wantedBy = [fsMountUnit];
             before = [fsMountUnit];
-            after = ["zfs-import-${utils.escapeSystemdPath vmCfg.zfs.pool}.service"];
+            after = [
+              "zfs-import-${utils.escapeSystemdPath vmCfg.zfs.pool}.service"
+              "zfs-mount.target"
+            ];
             unitConfig.DefaultDependencies = "no";
             serviceConfig = {
               Type = "oneshot";
@@ -181,30 +184,10 @@
         # TODO change once microvms are compatible with stage-1 systemd
         boot.initrd.systemd.enable = mkForce false;
 
-        # Create a firewall zone for the bridged traffic and secure vm traffic
         # TODO mkForce nftables
         networking.nftables.firewall = {
           zones = mkForce {
-            "${vmCfg.networking.mainLinkName}".interfaces = [vmCfg.networking.mainLinkName];
+            untrusted.interfaces = [vmCfg.networking.mainLinkName];
-            local-vms.interfaces = [config.extra.wireguard."${nodeName}-local-vms".linkName];
-          };
-
-          rules = mkForce {
-            "${vmCfg.networking.mainLinkName}-to-local" = {
-              from = [vmCfg.networking.mainLinkName];
-              to = ["local"];
-
-              inherit
-                (config.networking.firewall)
-                allowedTCPPorts
-                allowedUDPPorts
-                ;
-            };
-
-            local-vms-to-local = {
-              from = ["local-vms"];
-              to = ["local"];
-            };
           };
         };
 
@@ -215,7 +198,7 @@
               then "${config.networking.hostName}.local"
               else config.networking.fqdn;
             inherit (cfg.networking.wireguard) port;
-            openFirewallRules = ["${vmCfg.networking.mainLinkName}-to-local"];
+            openFirewallRules = ["untrusted"];
           };
           linkName = "local-vms";
           ipv4 = net.cidr.host vmCfg.id cfg.networking.wireguard.cidrv4;
@@ -402,21 +385,6 @@ in {
         ipv4 = net.cidr.host 1 cfg.networking.wireguard.cidrv4;
         ipv6 = net.cidr.host 1 cfg.networking.wireguard.cidrv6;
       };
-
-      # Create a firewall zone for the secure vm traffic
-      # TODO mkForce nftables
-      networking.nftables.firewall = {
-        zones = mkForce {
-          local-vms.interfaces = ["local-vms"];
-        };
-
-        rules = mkForce {
-          local-vms-to-local = {
-            from = ["local-vms"];
-            to = ["local"];
-          };
-        };
-      };
     }
     // extraLib.mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList microvmConfig vms)
   );

modules/wireguard.nix

@@ -21,6 +21,7 @@
     mapAttrsToList
     mdDoc
     mergeAttrs
+    mkForce
     mkIf
     mkOption
     optionalAttrs
@@ -133,11 +134,12 @@
       (isServer && wgCfg.server.openFirewall)
       [wgCfg.server.port];
 
+    # Open the port in the given nftables rule if specified
     # TODO mkForce nftables
-    networking.nftables.firewall.rules =
+    networking.nftables.firewall.rules = mkForce (
-      mkIf
+      optionalAttrs (isServer && wgCfg.server.openFirewallRules != [])
-      (isServer && wgCfg.server.openFirewallRules != [])
+      (genAttrs wgCfg.server.openFirewallRules (_: {allowedUDPPorts = [wgCfg.server.port];}))
-      (lib.mkForce (genAttrs wgCfg.server.openFirewallRules (_: {allowedUDPPorts = [wgCfg.server.port];})));
+    );
 
     rekey.secrets =
       concatAttrs (map