From 6d8f8ab2e3a8a29f00b2759c31eeadf6fe621fce Mon Sep 17 00:00:00 2001 From: oddlama Date: Sat, 20 May 2023 00:55:48 +0200 Subject: [PATCH] feat: add static microvm networking; allow cidrv4 and cidrv6 to be specified explicitly on wireguard networks to allow for simple access by other modules. --- hosts/ward/default.nix | 58 ------------- hosts/ward/net.nix | 24 ++++-- modules/microvms.nix | 183 +++++++++++++++++++++++++++-------------- modules/wireguard.nix | 34 +++++++- 4 files changed, 168 insertions(+), 131 deletions(-) diff --git a/hosts/ward/default.nix b/hosts/ward/default.nix index 4a589ff..890fa18 100644 --- a/hosts/ward/default.nix +++ b/hosts/ward/default.nix @@ -28,7 +28,6 @@ extra.microvms = { vms.test = { id = 11; - host = "test.local"; system = "x86_64-linux"; autostart = true; zfs = { @@ -39,61 +38,4 @@ }; }; }; - - #services.authelia.instances.main = { - # enable = true; - # settings = { - # theme = "dark"; - # log = { - # level = "info"; - # format = "text"; - # }; - # server = { - # host = "127.0.0.1"; - # port = 9091; - # }; - # session = { - # name = "session"; - # domain = "pas.sh"; - # }; - # authentication_backend.ldap = { - # implementation = "custom"; - # url = "ldap://127.0.0.1:3890"; - # base_dn = "dc=pas,dc=sh"; - # username_attribute = "uid"; - # additional_users_dn = "ou=people"; - # users_filter = "(&({username_attribute}={input})(objectclass=person))"; - # additional_groups_dn = "ou=groups"; - # groups_filter = "(member={dn})"; - # group_name_attribute = "cn"; - # mail_attribute = "mail"; - # display_name_attribute = "uid"; - # user = "uid=authelia,ou=people,dc=pas,dc=sh"; - # }; - # storage.local = { - # path = "/var/lib/authelia-${cfg.name}/db.sqlite3"; - # }; - # access_control = { - # default_policy = "deny"; - # }; - # notifier.smtp = rec { - # host = "smtp.fastmail.com"; - # port = 587; - # username = "a@example.com"; - # sender = "noreply@example.com"; - # startup_check_address = sender; - # disable_html_emails = true; - # }; - # identity_providers.oidc = { - # cors.allowed_origins_from_client_redirect_uris = true; - # cors.endpoints = [ - # "authorization" - # "introspection" - # "revocation" - # "token" - # "userinfo" - # ]; - # }; - # }; - #}; } diff --git a/hosts/ward/net.nix b/hosts/ward/net.nix index 0296336..fc83658 100644 --- a/hosts/ward/net.nix +++ b/hosts/ward/net.nix @@ -6,8 +6,8 @@ }: let inherit (config.lib.net) ip cidr; - net.lan.ipv4cidr = "192.168.100.1/24"; - net.lan.ipv6cidr = "fd00::1/64"; + lanCidrv4 = "192.168.100.0/24"; + lanCidrv6 = "fd00::/64"; in { networking.hostId = nodeSecrets.networking.hostId; @@ -55,7 +55,10 @@ in { linkConfig.RequiredForOnline = "routable"; }; "20-lan-self" = { - address = [net.lan.ipv4cidr net.lan.ipv6cidr]; + address = [ + (cidr.hostCidr 1 lanCidrv4) + (cidr.hostCidr 1 lanCidrv6) + ]; matchConfig.Name = "lan-self"; networkConfig = { IPForward = "yes"; @@ -64,7 +67,7 @@ in { }; # Announce a static prefix ipv6Prefixes = [ - {ipv6PrefixConfig.Prefix = cidr.canonicalize net.lan.ipv6cidr;} + {ipv6PrefixConfig.Prefix = lanCidrv6;} ]; # Delegate prefix from wan #dhcpPrefixDelegationConfig = { @@ -76,7 +79,7 @@ in { ipv6SendRAConfig = { EmitDNS = true; # TODO change to self later - #DNS = cidr.ip net.lan.ipv6cidr; + #DNS = cidr.host 1 net.lan.ipv6cidr; DNS = ["2606:4700:4700::1111" "2001:4860:4860::8888"]; }; linkConfig.RequiredForOnline = "routable"; @@ -160,14 +163,14 @@ in { subnet4 = [ { interface = "lan-self"; - subnet = cidr.canonicalize net.lan.ipv4cidr; + subnet = lanCidrv4; pools = [ - {pool = "${cidr.host 20 net.lan.ipv4cidr} - ${cidr.host (-6) net.lan.ipv4cidr}";} + {pool = "${cidr.host 20 lanCidrv4} - ${cidr.host (-6) lanCidrv4}";} ]; option-data = [ { name = "routers"; - data = cidr.ip net.lan.ipv4cidr; + data = cidr.host 1 lanCidrv4; } ]; } @@ -180,7 +183,10 @@ in { extra.microvms.networking = { baseMac = nodeSecrets.networking.interfaces.lan.mac; - host = cidr.ip net.lan.ipv4cidr; macvtapInterface = "lan"; + static = { + baseCidrv4 = lanCidrv4; + baseCidrv6 = lanCidrv6; + }; }; } diff --git a/modules/microvms.nix b/modules/microvms.nix index 7249548..b0de77b 100644 --- a/modules/microvms.nix +++ b/modules/microvms.nix @@ -32,6 +32,7 @@ cfg = config.extra.microvms; inherit (config.extra.microvms) vms; + inherit (config.lib) net; # Configuration for each microvm microvmConfig = vmName: vmCfg: { @@ -63,7 +64,7 @@ inherit (vmCfg) system; config = nodePath + "/microvms/${vmName}"; }; - mac = config.lib.net.mac.addPrivate vmCfg.id cfg.networking.baseMac; + mac = net.mac.addPrivate vmCfg.id cfg.networking.baseMac; in { # Allow children microvms to know which node is their parent specialArgs = @@ -87,7 +88,7 @@ id = "vm-${vmName}"; inherit mac; macvtap = { - link = cfg.macvtapInterface; + link = cfg.networking.macvtapInterface; mode = "bridge"; }; } @@ -124,19 +125,37 @@ gc.automatic = mkForce false; }; - extra.networking.renameInterfacesByMac.${vmCfg.linkName} = mac; + extra.networking.renameInterfacesByMac.${vmCfg.networking.mainLinkName} = mac; - systemd.network.networks = { - "10-${vmCfg.linkName}" = { - matchConfig.Name = vmCfg.linkName; - DHCP = "yes"; - networkConfig = { - IPv6PrivacyExtensions = "yes"; - IPv6AcceptRA = true; + systemd.network.networks."10-${vmCfg.networking.mainLinkName}" = + { + manual = {}; + dhcp = { + matchConfig.Name = vmCfg.networking.mainLinkName; + DHCP = "yes"; + networkConfig = { + IPv6PrivacyExtensions = "yes"; + IPv6AcceptRA = true; + }; + linkConfig.RequiredForOnline = "routable"; }; - linkConfig.RequiredForOnline = "routable"; - }; - }; + static = { + matchConfig.Name = vmCfg.networking.mainLinkName; + address = [ + vmCfg.networking.static.ipv4 + vmCfg.networking.static.ipv6 + ]; + gateway = [ + cfg.networking.host + ]; + networkConfig = { + IPv6PrivacyExtensions = "yes"; + IPv6AcceptRA = true; + }; + linkConfig.RequiredForOnline = "routable"; + }; + } + .${vmCfg.networking.mode}; # TODO change once microvms are compatible with stage-1 systemd boot.initrd.systemd.enable = mkForce false; @@ -144,13 +163,13 @@ # Create a firewall zone for the bridged traffic and secure vm traffic networking.nftables.firewall = { zones = lib.mkForce { - "${vmCfg.linkName}".interfaces = [vmCfg.linkName]; + "${vmCfg.networking.mainLinkName}".interfaces = [vmCfg.networking.mainLinkName]; "local-vms".interfaces = ["wg-local-vms"]; }; rules = lib.mkForce { - "${vmCfg.linkName}-to-local" = { - from = [vmCfg.linkName]; + "${vmCfg.networking.mainLinkName}-to-local" = { + from = [vmCfg.networking.mainLinkName]; to = ["local"]; }; @@ -161,24 +180,21 @@ }; }; - extra.wireguard."local-vms" = { + extra.wireguard."${nodeName}-local-vms" = { # We have a resolvable hostname / static ip, so all peers can directly communicate with us server = optionalAttrs (cfg.networking.host != null) { - inherit (vmCfg) host; + inherit (vmCfg.networking) host; port = 51829; - openFirewallInRules = ["${vmCfg.linkName}-to-local"]; + openFirewallInRules = ["${vmCfg.networking.mainLinkName}-to-local"]; }; - # We have no static hostname, so we must use a client-server architecture. + # If We don't have such guarantees, so we must use a client-server architecture. client = optionalAttrs (cfg.networking.host == null) { via = nodeName; keepalive = false; }; + cidrv4 = "${net.cidr.host vmCfg.id cfg.networking.wireguard.cidrv4}/32"; + cidrv6 = "${net.cidr.host vmCfg.id cfg.networking.wireguard.cidrv6}/128"; # TODO check error: addresses = ["10.22.22.2/30"]; - # TODO switch wg module to explicit v4 and v6 - addresses = [ - "${config.lib.net.cidr.host vmCfg.id cfg.networking.wireguard.netv4}/32" - "${config.lib.net.cidr.host vmCfg.id cfg.networking.wireguard.netv6}/128" - ]; }; }; }; @@ -194,18 +210,38 @@ in { options.extra.microvms = { networking = { baseMac = mkOption { - type = config.lib.net.types.mac; + type = net.types.mac; description = mdDoc '' This MAC address will be used as a base address to derive all MicroVM MAC addresses from. A good practise is to use the physical address of the macvtap interface. ''; }; + static = { + baseCidrv4 = mkOption { + type = net.types.cidrv4; + description = mdDoc '' + If a MicroVM is using static networking, and it hasn't defined a specific + address to use, its ipv4 address will be derived from this base address and its `id`. + ''; + }; + + baseCidrv6 = mkOption { + type = net.types.cidrv6; + description = mdDoc '' + If a MicroVM is using static networking, and it hasn't defined a specific + address to use, its ipv6 address will be derived from this base address and its `id`. + ''; + }; + }; + host = mkOption { type = types.str; + default = net.cidr.host 1 cfg.networking.static.baseCidrv4; description = mdDoc '' - The host as which this machine can be reached from other participants of the bridged macvtap network. - This can either be a resolvable hostname or an IP address. + The ip or resolveable hostname under which this machine can be reached from other + participants of the bridged macvtap network. Defaults to the first host + in the given static base ipv4 address range. ''; }; @@ -215,25 +251,24 @@ in { }; wireguard = { - netv4 = mkOption { - type = config.lib.net.types.cidrv4; + cidrv4 = mkOption { + type = net.types.cidrv4; description = mdDoc "The ipv4 network address range to use for internal vm traffic."; default = "172.31.0.0/24"; }; - netv6 = mkOption { - type = config.lib.net.types.cidrv6; + cidrv6 = mkOption { + type = net.types.cidrv6; description = mdDoc "The ipv6 network address range to use for internal vm traffic."; default = "fddd::/64"; }; }; - # TODO check plus no overflow }; vms = mkOption { default = {}; description = "Defines the actual vms and handles the necessary base setup for them."; - type = types.attrsOf (types.submodule { + type = types.attrsOf (types.submodule ({config, ...}: { options = { id = mkOption { type = @@ -254,6 +289,55 @@ in { ''; }; + networking = { + mode = mkOption { + type = types.enum ["dhcp" "static" "manual"]; + default = "static"; + description = "Determines how the main macvtap bridged network interface is configured this MicroVM."; + }; + + mainLinkName = mkOption { + type = types.str; + default = "wan"; + description = mdDoc "The main ethernet link name inside of the VM"; + }; + + static = { + ipv4 = mkOption { + type = net.types.ipv4-in cfg.networking.static.baseCidrv4; + default = net.cidr.host config.id cfg.networking.static.baseCidrv4; + description = mdDoc '' + The static ipv4 for this MicroVM. Only used if mode is static. + Defaults to the id-th host in the configured network range. + ''; + }; + + ipv6 = mkOption { + type = net.types.ipv6-in cfg.networking.static.baseCidrv6; + default = net.cidr.host config.id cfg.networking.static.baseCidrv6; + description = mdDoc '' + The static ipv6 for this MicroVM. Only used if mode is static. + Defaults to the id-th host in the configured network range. + ''; + }; + }; + + host = mkOption { + type = types.nullOr types.str; + default = + if config.networking.mode == "static" + then config.networking.static.ipv4 + else null; + description = mdDoc '' + The host as which this VM can be reached from other participants of the bridged macvtap network. + If this is null, the wireguard connection will use a client-server architecture with the host as the server. + Otherwise, all clients will communicate directly, meaning the host cannot listen to traffic. + + This can either be a resolvable hostname or an IP address. Defaults to the static ipv4 if given, else null. + ''; + }; + }; + zfs = { enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset"); @@ -279,33 +363,12 @@ in { description = mdDoc "Whether this VM should be started automatically with the host"; }; - # TODO allow configuring static ipv4 and ipv6 instead of dhcp? - # maybe create networking. namespace and have options = dhcpwithRA and static. - - linkName = mkOption { - type = types.str; - default = "wan"; - description = mdDoc "The main ethernet link name inside of the VM"; - }; - - host = mkOption { - type = types.nullOr types.str; - default = null; - description = mdDoc '' - The host as which this VM can be reached from other participants of the bridged macvtap network. - If this is unset, the wireguard connection will use a client-server architecture with the host as the server. - Otherwise, all clients will communicate directly, meaning the host cannot listen to traffic. - - This can either be a resolvable hostname or an IP address. - ''; - }; - system = mkOption { type = types.str; description = mdDoc "The system that this microvm should use"; }; }; - }); + })); }; }; @@ -321,16 +384,14 @@ in { ]; # Define a local wireguard server to communicate with vms securely - extra.wireguard."local-vms" = { + extra.wireguard."${nodeName}-local-vms" = { server = { inherit (cfg.networking) host; port = 51829; openFirewallInRules = ["lan-to-local"]; }; - addresses = [ - (config.lib.net.cidr.hostCidr 1 cfg.networking.wireguard.netv4) - (config.lib.net.cidr.hostCidr 1 cfg.networking.wireguard.netv6) - ]; + cidrv4 = net.cidr.hostCidr 1 cfg.networking.wireguard.cidrv4; + cidrv6 = net.cidr.hostCidr 1 cfg.networking.wireguard.cidrv6; }; } // extraLib.mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList microvmConfig vms) diff --git a/modules/wireguard.nix b/modules/wireguard.nix index 027c5b1..c528abf 100644 --- a/modules/wireguard.nix +++ b/modules/wireguard.nix @@ -177,7 +177,7 @@ }) wgCfg.server.externalPeers # All client nodes that have their via set to us. - ++ mapAttrsToList (clientNode: let + ++ map (clientNode: let clientCfg = wgCfgOf clientNode; in { wireguardPeerConfig = @@ -293,15 +293,43 @@ in { description = mdDoc "The order priority used when creating systemd netdev and network files."; }; + cidrv4 = mkOption { + type = + if config.client.via != null + then net.types.cidrv4-in nodes.${config.client.via}.config.extra.wireguard.${name}.cidrv4 + else net.types.cidrv4; + description = mdDoc '' + The ipv4 host address (with cidr mask) to configure for this interface. + The cidr mask determines this peers allowed address range as configured on other peers. + The mask should usually be fully restricted (/32) when no external clients are configured + and no other node uses this as a via. + ''; + }; + + cidrv6 = mkOption { + type = + if config.client.via != null + then net.types.cidrv6-in nodes.${config.client.via}.config.extra.wireguard.${name}.cidrv6 + else net.types.cidrv6; + description = mdDoc '' + The ipv6 host address (with cidr mask) to configure for this interface. + The cidr mask determines this peers allowed address range as configured on other peers. + The mask should usually be fully restricted (/128) when no external clients are configured + and no other node uses this as a via. + ''; + }; + addresses = mkOption { type = types.listOf ( if config.client.via != null then net.types.cidr-in nodes.${config.client.via}.config.extra.wireguard.${name}.addresses else net.types.cidr ); + default = [config.cidrv4 config.cidrv6]; description = mdDoc '' - The addresses to configure for this interface. Will automatically be added - as this peer's allowed addresses on all other peers. + The addresses (with cidr mask) to configure for this interface. + The cidr mask determines this peers allowed address range as configured on other peers. + By default this will just include {option}`cidrv4` and {option}`cidrv6` as configured. ''; }; };