forked from mirrors_public/oddlama_nix-config
feat: switch to DHCP based networking for microvms using mDNS for resolution
This commit is contained in:
oddlama
parent
0e3d881887
commit
e37601b486
8 changed files with 85 additions and 202 deletions
37
flake.lock
generated
37
flake.lock
generated
|
|
@ -119,11 +119,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1684472660,
|
||||
"narHash": "sha256-P4sR6f27FKoQuGnThELALUuJeu9mZ9Zh7/dYdaAd2ek=",
|
||||
"lastModified": 1684783210,
|
||||
"narHash": "sha256-hxRbwwBTu1G1u1EdI9nEo/n4HIsQIfNi+2BQ1nEoj/o=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "efb2016c8e6a91ea64e0604d69e332d8aceabb95",
|
||||
"rev": "f0b9f374bb42fdcd57baa7d4448ac5d4788226bd",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -210,11 +210,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1684484967,
|
||||
"narHash": "sha256-P3ftCqeJmDYS9LSr2gGC4XGGcp5vv8TOasJX6fVHWsw=",
|
||||
"lastModified": 1685019994,
|
||||
"narHash": "sha256-81o6SKZPALvib21hIOMx2lIhFSs0mRy0PfPvg0zsfTk=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "b9a52ad20e58ebd003444915e35e3dd2c18fc715",
|
||||
"rev": "d1f04b0f365a34896a37d9015637796537ec88a3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -260,11 +260,8 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1684191523,
|
||||
"narHash": "sha256-xIK3uQgSRqNLkEeD1vmRrpXjtYjpM7PK3liGMqAAPfc=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "fc98a8ca01ce1461d51728fd5df0c2ff609b5d09",
|
||||
"revCount": 486,
|
||||
"lastModified": 1685048145,
|
||||
"narHash": "sha256-IGvX/JZReujsF8dLJUr0+DdjplR5qeSOWKQbpXc9j9E=",
|
||||
"type": "git",
|
||||
"url": "file:///root/projects/microvm.nix"
|
||||
},
|
||||
|
|
@ -297,11 +294,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1683530131,
|
||||
"narHash": "sha256-R0RSqj6JdZfru2x/cM19KJMHsU52OjtyxI5cccd+uFc=",
|
||||
"lastModified": 1685000237,
|
||||
"narHash": "sha256-pm+2xP9g9sh6wapk1ulg7/1DdENkTNDB7Kx+6lwGs/k=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-generators",
|
||||
"rev": "10079333313ff62446e6f2b0e7c5231c7431d269",
|
||||
"rev": "05bef004794f352ea12475a89f3f55b4102c0728",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -312,11 +309,11 @@
|
|||
},
|
||||
"nixos-hardware": {
|
||||
"locked": {
|
||||
"lastModified": 1684169666,
|
||||
"narHash": "sha256-N5jrykeSxLVgvm3Dd3hZ38/XwM/jU+dltqlXgrGlYxk=",
|
||||
"lastModified": 1684899633,
|
||||
"narHash": "sha256-NtwerXX8UFsoNy6k+DukJMriWtEjQtMU/Urbff2O2Dg=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixos-hardware",
|
||||
"rev": "71ce85372a614d418d5e303dd5702a79d1545c04",
|
||||
"rev": "4cc688ee711159b9bcb5a367be44007934e1a49d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -389,11 +386,11 @@
|
|||
"nixpkgs-stable": "nixpkgs-stable"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1684195081,
|
||||
"narHash": "sha256-IKnQUSBhQTChFERxW2AzuauVpY1HRgeVzAjNMAA4B6I=",
|
||||
"lastModified": 1684842236,
|
||||
"narHash": "sha256-rYWsIXHvNhVQ15RQlBUv67W3YnM+Pd+DuXGMvCBq2IE=",
|
||||
"owner": "cachix",
|
||||
"repo": "pre-commit-hooks.nix",
|
||||
"rev": "96eabec58248ed8f4b0ad59e7ce9398018684fdc",
|
||||
"rev": "61e567d6497bc9556f391faebe5e410e6623217f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
|||
|
|
@ -61,22 +61,6 @@
|
|||
mode = "0700";
|
||||
}
|
||||
]
|
||||
++ lib.optionals (config.services.kea.dhcp4.enable || config.services.kea.dhcp6.enable) [
|
||||
{
|
||||
directory = "/var/lib/kea";
|
||||
user = "kea";
|
||||
group = "kea";
|
||||
mode = "0755";
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.services.gitea.enable [
|
||||
{
|
||||
directory = "/var/lib/gitea";
|
||||
user = "gitea";
|
||||
group = "gitea";
|
||||
mode = "0755";
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.security.acme.acceptTerms [
|
||||
{
|
||||
directory = "/var/lib/acme";
|
||||
|
|
@ -101,13 +85,29 @@
|
|||
mode = "0750";
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.services.opendkim.enable [
|
||||
++ lib.optionals config.services.postgresql.enable [
|
||||
{
|
||||
directory = "/var/lib/postgresql";
|
||||
user = "postgres";
|
||||
group = "postgres";
|
||||
mode = "0755";
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.services.gitea.enable [
|
||||
{
|
||||
directory = "/var/lib/gitea";
|
||||
user = "gitea";
|
||||
group = "gitea";
|
||||
mode = "0755";
|
||||
}
|
||||
]
|
||||
++ lib.optionals config.services.kanidm.enableServer [
|
||||
{
|
||||
directory = "/var/lib/kanidm";
|
||||
user = "kanidm";
|
||||
group = "kanidm";
|
||||
mode = "0755";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
"8.8.8.8"
|
||||
"2001:4860:4860::8844"
|
||||
];
|
||||
llmnr = "true"; # Microsoft's version of mDNS
|
||||
llmnr = "false";
|
||||
extraConfig = ''
|
||||
Domains=~.
|
||||
MulticastDNS=true
|
||||
|
|
|
|||
|
|
@ -13,14 +13,20 @@
|
|||
"10-lan1" = {
|
||||
DHCP = "yes";
|
||||
matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.lan1.mac;
|
||||
networkConfig.IPv6PrivacyExtensions = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
MulticastDNS = true;
|
||||
};
|
||||
dhcpV4Config.RouteMetric = 10;
|
||||
dhcpV6Config.RouteMetric = 10;
|
||||
};
|
||||
"10-wlan1" = {
|
||||
DHCP = "yes";
|
||||
matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.wlan1.mac;
|
||||
networkConfig.IPv6PrivacyExtensions = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
MulticastDNS = true;
|
||||
};
|
||||
dhcpV4Config.RouteMetric = 40;
|
||||
dhcpV6Config.RouteMetric = 40;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -83,18 +83,6 @@ in {
|
|||
}: {
|
||||
rekey.hostPubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN2TxWynLb8V9SP45kFqsoCWhe/dG8N1xWNuJG5VQndq";
|
||||
|
||||
rekey.secrets."kanidm-self-signed.crt" = {
|
||||
file = ./secrets/kanidm-self-signed.crt.age;
|
||||
mode = "440";
|
||||
owner = "nginx";
|
||||
group = "kanidm";
|
||||
};
|
||||
rekey.secrets."kanidm-self-signed.key" = {
|
||||
file = ./secrets/kanidm-self-signed.key.age;
|
||||
mode = "440";
|
||||
owner = "nginx";
|
||||
group = "kanidm";
|
||||
};
|
||||
rekey.secrets."dhparams.pem" = {
|
||||
# TODO make own?
|
||||
file = ../zackbiene/secrets/dhparams.pem.age;
|
||||
|
|
@ -102,10 +90,6 @@ in {
|
|||
group = "nginx";
|
||||
};
|
||||
|
||||
networking.hosts = {
|
||||
"192.168.100.12" = [auth.domain];
|
||||
};
|
||||
|
||||
rekey.secrets.acme-credentials = {
|
||||
file = ./secrets/acme-credentials.age;
|
||||
mode = "440";
|
||||
|
|
@ -125,10 +109,6 @@ in {
|
|||
};
|
||||
users.groups.acme.members = ["nginx"];
|
||||
|
||||
# TODO needed in my current testing network that has no ipv6 connectivity
|
||||
# TODO but these should use fallback......... something's wrong
|
||||
systemd.network.networks."10-wan".networkConfig.DNS = ["1.1.1.1" "8.8.8.8"];
|
||||
|
||||
# TODO reload nginx when acme is renewed
|
||||
|
||||
# TODO make default nginx config in core to reduce boilerplate?
|
||||
|
|
@ -195,18 +175,16 @@ in {
|
|||
local-vms-to-local.allowedTCPPorts = [8300];
|
||||
};
|
||||
|
||||
# systemd.services.kanidm = let
|
||||
# cfg = config.services.kanidm;
|
||||
# certName = config.services.nginx.virtualHosts.${cfg.serverSettings.domain}.useACMEHost;
|
||||
# in {
|
||||
# requires = [ "acme-finished-${certName}.target" ];
|
||||
# serviceConfig.LoadCredential = let
|
||||
# certDir = config.security.acme.certs.${certName}.directory;
|
||||
# in [
|
||||
# "fullchain.pem:${certDir}/fullchain.pem"
|
||||
# "key.pem:${certDir}/key.pem"
|
||||
# ];
|
||||
# };
|
||||
rekey.secrets."kanidm-self-signed.crt" = {
|
||||
file = ./secrets/kanidm-self-signed.crt.age;
|
||||
mode = "440";
|
||||
group = "kanidm";
|
||||
};
|
||||
rekey.secrets."kanidm-self-signed.key" = {
|
||||
file = ./secrets/kanidm-self-signed.key.age;
|
||||
mode = "440";
|
||||
group = "kanidm";
|
||||
};
|
||||
|
||||
services.kanidm = {
|
||||
enableServer = true;
|
||||
|
|
@ -221,7 +199,11 @@ in {
|
|||
bindaddress = "${config.extra.wireguard."${parentNodeName}-local-vms".ipv4}:8300";
|
||||
trust_x_forward_for = true;
|
||||
};
|
||||
};
|
||||
|
||||
environment.systemPackages = [pkgs.kanidm];
|
||||
|
||||
services.kanidm = {
|
||||
enableClient = true;
|
||||
clientSettings = {
|
||||
uri = config.services.kanidm.serverSettings.origin;
|
||||
|
|
@ -229,7 +211,5 @@ in {
|
|||
verify_hostnames = true;
|
||||
};
|
||||
};
|
||||
|
||||
environment.systemPackages = [pkgs.kanidm];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
inherit (config.lib.net) cidr;
|
||||
|
||||
lanCidrv4 = "192.168.100.0/24";
|
||||
lanCidrv6 = "fd00::/64";
|
||||
lanCidrv6 = "fd10::/64";
|
||||
in {
|
||||
networking.hostId = config.repo.secrets.local.networking.hostId;
|
||||
|
||||
|
|
@ -63,6 +63,7 @@ in {
|
|||
IPForward = "yes";
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
IPv6SendRA = true;
|
||||
MulticastDNS = true;
|
||||
};
|
||||
# Announce a static prefix
|
||||
ipv6Prefixes = [
|
||||
|
|
@ -83,13 +84,6 @@ in {
|
|||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
# Remaining macvtap interfaces should not be touched.
|
||||
"90-macvtap-no-ll" = {
|
||||
matchConfig.Kind = "macvtap";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
linkConfig.ActivationPolicy = "manual";
|
||||
linkConfig.Unmanaged = "yes";
|
||||
};
|
||||
};
|
||||
|
||||
# TODO mkForce nftables
|
||||
|
|
@ -165,7 +159,7 @@ in {
|
|||
interface = "lan-self";
|
||||
subnet = lanCidrv4;
|
||||
pools = [
|
||||
{pool = "${cidr.host 40 lanCidrv4} - ${cidr.host (-6) lanCidrv4}";}
|
||||
{pool = "${cidr.host 20 lanCidrv4} - ${cidr.host (-6) lanCidrv4}";}
|
||||
];
|
||||
option-data = [
|
||||
{
|
||||
|
|
@ -184,10 +178,6 @@ in {
|
|||
extra.microvms.networking = {
|
||||
baseMac = config.repo.secrets.local.networking.interfaces.lan.mac;
|
||||
macvtapInterface = "lan";
|
||||
static = {
|
||||
baseCidrv4 = lanCidrv4;
|
||||
baseCidrv6 = lanCidrv6;
|
||||
};
|
||||
wireguard.openFirewallRules = ["lan-to-local"];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ in {
|
|||
"10-lan1" = {
|
||||
DHCP = "yes";
|
||||
matchConfig.MACAddress = config.repo.secrets.local.networking.interfaces.lan1.mac;
|
||||
networkConfig.IPv6PrivacyExtensions = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
MulticastDNS = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
"10-wlan1" = {
|
||||
|
|
|
|||
|
|
@ -166,35 +166,16 @@
|
|||
# agenix activation had an error, but this is not trivial.
|
||||
${wgConfig}.linkConfig.RequiredForOnline = "no";
|
||||
|
||||
"10-${vmCfg.networking.mainLinkName}" =
|
||||
{
|
||||
manual = {};
|
||||
dhcp = {
|
||||
matchConfig.Name = vmCfg.networking.mainLinkName;
|
||||
DHCP = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
static = {
|
||||
matchConfig.Name = vmCfg.networking.mainLinkName;
|
||||
address = [
|
||||
"${vmCfg.networking.static.ipv4}/${toString (net.cidr.length cfg.networking.static.baseCidrv4)}"
|
||||
"${vmCfg.networking.static.ipv6}/${toString (net.cidr.length cfg.networking.static.baseCidrv6)}"
|
||||
];
|
||||
gateway = [
|
||||
cfg.networking.host
|
||||
];
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
}
|
||||
.${vmCfg.networking.mode};
|
||||
"10-${vmCfg.networking.mainLinkName}" = {
|
||||
matchConfig.MACAddress = mac;
|
||||
DHCP = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
MulticastDNS = true;
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
};
|
||||
|
||||
# TODO change once microvms are compatible with stage-1 systemd
|
||||
|
|
@ -228,17 +209,13 @@
|
|||
};
|
||||
|
||||
extra.wireguard."${nodeName}-local-vms" = {
|
||||
# We have a resolvable hostname / static ip, so all peers can directly communicate with us
|
||||
server = optionalAttrs (cfg.networking.host != null) {
|
||||
inherit (vmCfg.networking) host;
|
||||
server = {
|
||||
host =
|
||||
if config.networking.domain == null
|
||||
then "${config.networking.hostName}.local"
|
||||
else config.networking.fqdn;
|
||||
inherit (cfg.networking.wireguard) port;
|
||||
openFirewallRules = ["${vmCfg.networking.mainLinkName}-to-local"];
|
||||
reservedAddresses = [cfg.networking.wireguard.cidrv4 cfg.networking.wireguard.cidrv6];
|
||||
};
|
||||
# If We don't have such guarantees, so we must use a client-server architecture.
|
||||
client = optionalAttrs (cfg.networking.host == null) {
|
||||
via = nodeName;
|
||||
keepalive = false;
|
||||
};
|
||||
linkName = "local-vms";
|
||||
ipv4 = net.cidr.host vmCfg.id cfg.networking.wireguard.cidrv4;
|
||||
|
|
@ -271,34 +248,6 @@ in {
|
|||
'';
|
||||
};
|
||||
|
||||
static = {
|
||||
baseCidrv4 = mkOption {
|
||||
type = net.types.cidrv4;
|
||||
description = mdDoc ''
|
||||
If a MicroVM is using static networking, and it hasn't defined a specific
|
||||
address to use, its ipv4 address will be derived from this base address and its `id`.
|
||||
'';
|
||||
};
|
||||
|
||||
baseCidrv6 = mkOption {
|
||||
type = net.types.cidrv6;
|
||||
description = mdDoc ''
|
||||
If a MicroVM is using static networking, and it hasn't defined a specific
|
||||
address to use, its ipv6 address will be derived from this base address and its `id`.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
host = mkOption {
|
||||
type = types.str;
|
||||
default = net.cidr.host 1 cfg.networking.static.baseCidrv4;
|
||||
description = mdDoc ''
|
||||
The ip or resolveable hostname under which this machine can be reached from other
|
||||
participants of the bridged macvtap network. Defaults to the first host
|
||||
in the given static base ipv4 address range.
|
||||
'';
|
||||
};
|
||||
|
||||
macvtapInterface = mkOption {
|
||||
type = types.str;
|
||||
description = mdDoc "The macvtap interface to which MicroVMs should be attached";
|
||||
|
|
@ -380,62 +329,16 @@ in {
|
|||
description = mdDoc ''
|
||||
A unique id for this VM. It will be used to derive a MAC address from the host's
|
||||
base MAC, and may be used as a stable id by your MicroVM config if necessary.
|
||||
|
||||
Ids don't need to be contiguous. It is recommended to use small numbers here to not
|
||||
overflow any offset calculations. Consider that this is used for example to determine a
|
||||
static ip-address by means of (baseIp + vm.id) for a wireguard network. That's also
|
||||
why id 1 is reserved for the host. While this is usually checked to be in-range,
|
||||
it might still be a good idea to assign greater ids with care.
|
||||
Ids don't need to be contiguous.
|
||||
'';
|
||||
};
|
||||
|
||||
networking = {
|
||||
mode = mkOption {
|
||||
type = types.enum ["dhcp" "static" "manual"];
|
||||
default = "static";
|
||||
description = "Determines how the main macvtap bridged network interface is configured this MicroVM.";
|
||||
};
|
||||
|
||||
mainLinkName = mkOption {
|
||||
type = types.str;
|
||||
default = "wan";
|
||||
description = mdDoc "The main ethernet link name inside of the VM";
|
||||
};
|
||||
|
||||
static = {
|
||||
ipv4 = mkOption {
|
||||
type = net.types.ipv4-in cfg.networking.static.baseCidrv4;
|
||||
default = net.cidr.host config.id cfg.networking.static.baseCidrv4;
|
||||
description = mdDoc ''
|
||||
The static ipv4 for this MicroVM. Only used if mode is static.
|
||||
Defaults to the id-th host in the configured network range.
|
||||
'';
|
||||
};
|
||||
|
||||
ipv6 = mkOption {
|
||||
type = net.types.ipv6-in cfg.networking.static.baseCidrv6;
|
||||
default = net.cidr.host config.id cfg.networking.static.baseCidrv6;
|
||||
description = mdDoc ''
|
||||
The static ipv6 for this MicroVM. Only used if mode is static.
|
||||
Defaults to the id-th host in the configured network range.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
host = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default =
|
||||
if config.networking.mode == "static"
|
||||
then config.networking.static.ipv4
|
||||
else null;
|
||||
description = mdDoc ''
|
||||
The host as which this VM can be reached from other participants of the bridged macvtap network.
|
||||
If this is null, the wireguard connection will use a client-server architecture with the host as the server.
|
||||
Otherwise, all clients will communicate directly, meaning the host cannot listen to traffic.
|
||||
|
||||
This can either be a resolvable hostname or an IP address. Defaults to the static ipv4 if given, else null.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
zfs = {
|
||||
|
|
@ -488,8 +391,12 @@ in {
|
|||
# Define a local wireguard server to communicate with vms securely
|
||||
extra.wireguard."${nodeName}-local-vms" = {
|
||||
server = {
|
||||
inherit (cfg.networking) host;
|
||||
host =
|
||||
if config.networking.domain == null
|
||||
then "${config.networking.hostName}.local"
|
||||
else config.networking.fqdn;
|
||||
inherit (cfg.networking.wireguard) openFirewallRules port;
|
||||
reservedAddresses = [cfg.networking.wireguard.cidrv4 cfg.networking.wireguard.cidrv6];
|
||||
};
|
||||
linkName = "local-vms";
|
||||
ipv4 = net.cidr.host 1 cfg.networking.wireguard.cidrv4;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue