feat: refactor and integrate wireguard module into microvm module

This commit is contained in:
oddlama 2023-05-19 21:10:16 +02:00
parent e5f3ffd288
commit 78cdcd3c69
No known key found for this signature in database
GPG key ID: 14EFE510775FE39A
10 changed files with 385 additions and 256 deletions

View file

@ -25,11 +25,13 @@
mkMerge
mkOption
optional
optionalAttrs
recursiveUpdate
types
;
cfg = config.extra.microvms;
inherit (config.extra.microvms) vms;
# Configuration for each microvm
microvmConfig = vmName: vmCfg: {
@ -61,8 +63,16 @@
inherit (vmCfg) system;
config = nodePath + "/microvms/${vmName}";
};
mac = config.lib.net.mac.addPrivate vmCfg.id cfg.networking.baseMac;
in {
inherit (node) pkgs specialArgs;
# Allow children microvms to know which node is their parent
specialArgs =
{
parentNode = config;
parentNodeName = nodeName;
}
// node.specialArgs;
inherit (node) pkgs;
inherit (vmCfg) autostart;
config = {
imports = [microvm.microvm] ++ node.imports;
@ -75,11 +85,11 @@
{
type = "macvtap";
id = "vm-${vmName}";
inherit mac;
macvtap = {
link = vmCfg.macvtap;
link = cfg.macvtapInterface;
mode = "bridge";
};
inherit (vmCfg) mac;
}
];
@ -114,7 +124,7 @@
gc.automatic = mkForce false;
};
extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac;
extra.networking.renameInterfacesByMac.${vmCfg.linkName} = mac;
systemd.network.networks = {
"10-${vmCfg.linkName}" = {
@ -130,6 +140,46 @@
# TODO change once microvms are compatible with stage-1 systemd
boot.initrd.systemd.enable = mkForce false;
# Create a firewall zone for the bridged traffic and secure vm traffic
networking.nftables.firewall = {
zones = lib.mkForce {
"${vmCfg.linkName}".interfaces = [vmCfg.linkName];
"local-vms".interfaces = ["wg-local-vms"];
};
rules = lib.mkForce {
"${vmCfg.linkName}-to-local" = {
from = [vmCfg.linkName];
to = ["local"];
};
local-vms-to-local = {
from = ["wg-local-vms"];
to = ["local"];
};
};
};
extra.wireguard."local-vms" = {
# We have a resolvable hostname / static ip, so all peers can directly communicate with us
server = optionalAttrs (cfg.networking.host != null) {
inherit (vmCfg) host;
port = 51829;
openFirewallInRules = ["${vmCfg.linkName}-to-local"];
};
# We have no static hostname, so we must use a client-server architecture.
client = optionalAttrs (cfg.networking.host == null) {
via = nodeName;
keepalive = false;
};
# TODO check error: addresses = ["10.22.22.2/30"];
# TODO switch wg module to explicit v4 and v6
addresses = [
"${config.lib.net.cidr.host vmCfg.id cfg.networking.wireguard.netv4}/32"
"${config.lib.net.cidr.host vmCfg.id cfg.networking.wireguard.netv6}/128"
];
};
};
};
};
@ -138,78 +188,151 @@ in {
# Add the host module, but only enable if it necessary
microvm.host
# This is opt-out, so we can't put this into the mkIf below
{microvm.host.enable = cfg != {};}
{microvm.host.enable = vms != {};}
];
options.extra.microvms = mkOption {
default = {};
description = "Handles the necessary base setup for MicroVMs.";
type = types.attrsOf (types.submodule {
options = {
zfs = {
enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset");
options.extra.microvms = {
networking = {
baseMac = mkOption {
type = config.lib.net.types.mac;
description = mdDoc ''
This MAC address will be used as a base address to derive all MicroVM MAC addresses from.
A good practise is to use the physical address of the macvtap interface.
'';
};
pool = mkOption {
type = types.str;
description = mdDoc "The host's zfs pool on which the dataset resides";
};
host = mkOption {
type = types.str;
description = mdDoc ''
The host as which this machine can be reached from other participants of the bridged macvtap network.
This can either be a resolvable hostname or an IP address.
'';
};
dataset = mkOption {
type = types.str;
description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
};
macvtapInterface = mkOption {
type = types.str;
description = mdDoc "The macvtap interface to which MicroVMs should be attached";
};
mountpoint = mkOption {
type = types.str;
description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtofs as /persist in the vm)";
};
wireguard = {
netv4 = mkOption {
type = config.lib.net.types.cidrv4;
description = mdDoc "The ipv4 network address range to use for internal vm traffic.";
default = "172.31.0.0/24";
};
autostart = mkOption {
type = types.bool;
default = false;
description = mdDoc "Whether this VM should be started automatically with the host";
};
linkName = mkOption {
type = types.str;
default = "wan";
description = mdDoc "The main ethernet link name inside of the VM";
};
mac = mkOption {
type = config.lib.net.types.mac;
description = mdDoc "The MAC address to assign to this VM";
};
macvtap = mkOption {
type = types.str;
description = mdDoc "The macvtap interface to attach to";
};
system = mkOption {
type = types.str;
description = mdDoc "The system that this microvm should use";
netv6 = mkOption {
type = config.lib.net.types.cidrv6;
description = mdDoc "The ipv6 network address range to use for internal vm traffic.";
default = "fddd::/64";
};
};
});
# TODO check plus no overflow
};
vms = mkOption {
default = {};
description = "Defines the actual vms and handles the necessary base setup for them.";
type = types.attrsOf (types.submodule {
options = {
id = mkOption {
type =
types.addCheck types.int (x: x > 1)
// {
name = "positiveInt1";
description = "positive integer greater than 1";
};
description = mdDoc ''
A unique id for this VM. It will be used to derive a MAC address from the host's
base MAC, and may be used as a stable id by your MicroVM config if necessary.
Ids don't need to be contiguous. It is recommended to use small numbers here to not
overflow any offset calculations. Consider that this is used for example to determine a
static ip-address by means of (baseIp + vm.id) for a wireguard network. That's also
why id 1 is reserved for the host. While this is usually checked to be in-range,
it might still be a good idea to assign greater ids with care.
'';
};
zfs = {
enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset");
pool = mkOption {
type = types.str;
description = mdDoc "The host's zfs pool on which the dataset resides";
};
dataset = mkOption {
type = types.str;
description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
};
mountpoint = mkOption {
type = types.str;
description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtofs as /persist in the vm)";
};
};
autostart = mkOption {
type = types.bool;
default = false;
description = mdDoc "Whether this VM should be started automatically with the host";
};
# TODO allow configuring static ipv4 and ipv6 instead of dhcp?
# maybe create networking. namespace and have options = dhcpwithRA and static.
linkName = mkOption {
type = types.str;
default = "wan";
description = mdDoc "The main ethernet link name inside of the VM";
};
host = mkOption {
type = types.nullOr types.str;
default = null;
description = mdDoc ''
The host as which this VM can be reached from other participants of the bridged macvtap network.
If this is unset, the wireguard connection will use a client-server architecture with the host as the server.
Otherwise, all clients will communicate directly, meaning the host cannot listen to traffic.
This can either be a resolvable hostname or an IP address.
'';
};
system = mkOption {
type = types.str;
description = mdDoc "The system that this microvm should use";
};
};
});
};
};
config = mkIf (cfg != {}) (
config = mkIf (vms != {}) (
{
assertions = let
duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
duplicateIds = extraLib.duplicates (mapAttrsToList (_: vmCfg: toString vmCfg.id) vms);
in [
{
assertion = duplicateMacs == [];
message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
assertion = duplicateIds == [];
message = "Duplicate MicroVM ids: ${concatStringsSep ", " duplicateIds}";
}
];
# Define a local wireguard server to communicate with vms securely
extra.wireguard."local-vms" = {
server = {
inherit (cfg.networking) host;
port = 51829;
openFirewallInRules = ["lan-to-local"];
};
addresses = [
(config.lib.net.cidr.hostCidr 1 cfg.networking.wireguard.netv4)
(config.lib.net.cidr.hostCidr 1 cfg.networking.wireguard.netv6)
];
};
}
// lib.genAttrs ["disko" "microvm" "systemd"]
(attr:
mkMerge (map
(c: c.${attr})
(mapAttrsToList microvmConfig cfg)))
// extraLib.mergeToplevelConfigs ["disko" "microvm" "systemd"] (mapAttrsToList microvmConfig vms)
);
}

View file

@ -17,14 +17,13 @@
concatStringsSep
filter
filterAttrs
genAttrs
head
mapAttrsToList
mdDoc
mergeAttrs
mkIf
mkOption
mkEnableOption
net
optionalAttrs
optionals
splitString
@ -35,57 +34,102 @@
(extraLib)
concatAttrs
duplicates
mergeToplevelConfigs
;
inherit (config.lib) net;
cfg = config.extra.wireguard;
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
# TODO use netlib types!!!!!!
configForNetwork = wgName: wgCfg: let
inherit
(extraLib.wireguard wgName)
associatedNodes
associatedServerNodes
associatedClientNodes
externalPeerName
externalPeerNamesRaw
peerPresharedKeyPath
peerPresharedKeySecret
peerPrivateKeyPath
peerPrivateKeySecret
peerPublicKeyPath
usedAddresses
;
isServer = wgCfg.server.host != null;
isClient = wgCfg.client.via != null;
filterSelf = filter (x: x != nodeName);
wgCfgOf = node: nodes.${node}.config.extra.wireguard.${wgName};
# All nodes that use our node as the via into the wireguard network
ourClientNodes =
optionals wgCfg.server.enable
(filter (n: (wgCfgOf n).via == nodeName) associatedClientNodes);
optionals isServer
(filter (n: (wgCfgOf n).client.via == nodeName) associatedClientNodes);
# The list of peers that we have to know the psk to.
# The list of peers for which we have to know the psk.
neededPeers =
if wgCfg.server.enable
if isServer
then
# Other servers in the same network
filterSelf associatedServerNodes
# Our external peers
++ map externalPeerName (attrNames wgCfg.server.externalPeers)
# Our clients
++ ourClientNodes
else [wgCfg.via];
in {
secrets =
concatAttrs (map (other: {
${peerPresharedKeySecret nodeName other}.file = peerPresharedKeyPath nodeName other;
})
neededPeers)
// {
${peerPrivateKeySecret nodeName}.file = peerPrivateKeyPath nodeName;
};
else [wgCfg.client.via];
netdevs."${wgCfg.priority}-${wgName}" = {
# Figure out if there are duplicate peers or addresses so we can
# make an assertion later.
duplicatePeers = duplicates externalPeerNamesRaw;
duplicateAddrs = duplicates (map (x: head (splitString "/" x)) usedAddresses);
# Adds context information to the assertions for this network
assertionPrefix = "Wireguard network '${wgName}' on '${nodeName}'";
in {
assertions = [
{
assertion = any (n: (wgCfgOf n).server.host != null) associatedNodes;
message = "${assertionPrefix}: At least one node in a network must be a server.";
}
{
assertion = duplicatePeers == [];
message = "${assertionPrefix}: Multiple definitions for external peer(s):${concatMapStrings (x: " '${x}'") duplicatePeers}";
}
{
assertion = duplicateAddrs == [];
message = "${assertionPrefix}: Addresses used multiple times: ${concatStringsSep ", " duplicateAddrs}";
}
{
assertion = isServer != isClient;
message = "${assertionPrefix}: A node must either be a server (define server.host) or a client (define client.via).";
}
{
assertion = isClient -> ((wgCfgOf wgCfg.client.via).server.host != null);
message = "${assertionPrefix}: The specified via node '${wgCfg.client.via}' must be a wireguard server.";
}
# TODO externalPeers != {} -> ip forwarding
# TODO no overlapping cidrs in (external peers + peers using via = this).
# TODO no overlapping cidrs between server nodes
];
networking.firewall.allowedUDPPorts =
mkIf
(isServer && wgCfg.server.openFirewall)
[wgCfg.server.port];
networking.nftables.firewall.rules =
mkIf
(isServer && wgCfg.server.openFirewallInRules != [])
(genAttrs wgCfg.server.openFirewallInRules (_: {allowedUDPPorts = [wgCfg.server.port];}));
rekey.secrets =
concatAttrs (map
(other: {${peerPresharedKeySecret nodeName other}.file = peerPresharedKeyPath nodeName other;})
neededPeers)
// {${peerPrivateKeySecret nodeName}.file = peerPrivateKeyPath nodeName;};
systemd.network.netdevs."${toString wgCfg.priority}-${wgName}" = {
netdevConfig = {
Kind = "wireguard";
Name = "${wgName}";
@ -95,17 +139,17 @@
{
PrivateKeyFile = config.rekey.secrets.${peerPrivateKeySecret nodeName}.path;
}
// optionalAttrs wgCfg.server.enable {
// optionalAttrs isServer {
ListenPort = wgCfg.server.port;
};
wireguardPeers =
if wgCfg.server.enable
if isServer
then
# Always include all other server nodes.
map (serverNode: let
snCfg = wgCfgOf serverNode;
in {
wireguardPeerConfig = {
map (serverNode: {
wireguardPeerConfig = let
snCfg = wgCfgOf serverNode;
in {
PublicKey = builtins.readFile (peerPublicKeyPath serverNode);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName serverNode}.path;
# The allowed ips of a server node are it's own addreses,
@ -117,7 +161,8 @@
# ++ map (n: (wgCfgOf n).addresses) snCfg.ourClientNodes;
Endpoint = "${snCfg.server.host}:${toString snCfg.server.port}";
};
}) (filterSelf associatedServerNodes)
})
(filterSelf associatedServerNodes)
# All our external peers
++ mapAttrsToList (extPeer: allowedIPs: let
peerName = externalPeerName extPeer;
@ -126,18 +171,24 @@
PublicKey = builtins.readFile (peerPublicKeyPath peerName);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName peerName}.path;
AllowedIPs = allowedIPs;
# Connections to external peers should always be kept alive
PersistentKeepalive = 25;
};
})
wgCfg.server.externalPeers
# All client nodes that have their via set to us.
++ mapAttrsToList (clientNode: {
wireguardPeerConfig = {
PublicKey = builtins.readFile (peerPublicKeyPath clientNode);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName clientNode}.path;
AllowedIPs = (wgCfgOf clientNode).addresses;
PersistentKeepalive = 25;
};
++ mapAttrsToList (clientNode: let
clientCfg = wgCfgOf clientNode;
in {
wireguardPeerConfig =
{
PublicKey = builtins.readFile (peerPublicKeyPath clientNode);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName clientNode}.path;
AllowedIPs = clientCfg.addresses;
}
// optionalAttrs clientCfg.keepalive {
PersistentKeepalive = 25;
};
})
ourClientNodes
else
@ -145,15 +196,16 @@
[
{
wireguardPeerConfig = {
PublicKey = builtins.readFile (peerPublicKeyPath wgCfg.via);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName wgCfg.via}.path;
AllowedIPs = (wgCfgOf wgCfg.via).addresses;
PublicKey = builtins.readFile (peerPublicKeyPath wgCfg.client.via);
PresharedKeyFile = config.rekey.secrets.${peerPresharedKeySecret nodeName wgCfg.client.via}.path;
# TODO this should be 0.0.0.0 if the client wants to route all traffic
AllowedIPs = (wgCfgOf wgCfg.client.via).addresses;
};
}
];
};
networks."${wgCfg.priority}-${wgName}" = {
systemd.network.networks."${toString wgCfg.priority}-${wgName}" = {
matchConfig.Name = wgName;
networkConfig.Address = wgCfg.addresses;
};
@ -162,14 +214,17 @@ in {
options.extra.wireguard = mkOption {
default = {};
description = "Configures wireguard networks via systemd-networkd.";
type = types.attrsOf (types.submodule {
type = types.lazyAttrsOf (types.submodule ({
config,
name,
...
}: {
options = {
server = {
enable = mkEnableOption (mdDoc "wireguard server");
host = mkOption {
type = types.str;
description = mdDoc "The hostname or ip address which other peers can use to reach this host.";
default = null;
type = types.nullOr types.str;
description = mdDoc "The hostname or ip address which other peers can use to reach this host. No server funnctionality will be activated if set to null.";
};
port = mkOption {
@ -181,11 +236,17 @@ in {
openFirewall = mkOption {
default = false;
type = types.bool;
description = mdDoc "Whether to open the firewall for the specified `listenPort`, if {option}`listen` is `true`.";
description = mdDoc "Whether to open the firewall for the specified {option}`port`.";
};
openFirewallInRules = mkOption {
default = [];
type = types.listOf types.str;
description = mdDoc "The {option}`port` will be opened for all of the given rules in the nftable-firewall.";
};
externalPeers = mkOption {
type = types.attrsOf (types.listOf types.str);
type = types.attrsOf (types.listOf (net.types.cidr-in config.addresses));
default = {};
example = {my-android-phone = ["10.0.0.97/32"];};
description = mdDoc ''
@ -199,82 +260,55 @@ in {
};
};
client = {
via = mkOption {
default = null;
type = types.nullOr types.str;
description = mdDoc ''
The server node via which to connect to the network.
No client functionality will be activated if set to null.
'';
};
keepalive = mkOption {
default = true;
type = types.bool;
description = mdDoc "Whether to keep this connection alive using PersistentKeepalive. Set to false only for networks where client and server IPs are stable.";
};
# TODO one option for allowing it, but also one to allow defining two
# profiles / interfaces that can be activated manually.
#routeAllTraffic = mkOption {
# default = false;
# type = types.bool;
# description = mdDoc ''
# Whether to allow routing all traffic through the via server.
# '';
#};
};
priority = mkOption {
default = "20";
type = types.str;
default = 40;
type = types.int;
description = mdDoc "The order priority used when creating systemd netdev and network files.";
};
via = mkOption {
default = null;
type = types.uniq (types.nullOr types.str);
description = mdDoc ''
The server node via which to connect to the network.
This must defined if and only if this node is not a server.
'';
};
addresses = mkOption {
type = types.listOf types.str;
type = types.listOf (
if config.client.via != null
then net.types.cidr-in nodes.${config.client.via}.config.extra.wireguard.${name}.addresses
else net.types.cidr
);
description = mdDoc ''
The addresses to configure for this interface. Will automatically be added
as this peer's allowed addresses to all other peers.
as this peer's allowed addresses on all other peers.
'';
};
};
});
}));
};
config = mkIf (cfg != {}) (let
networkCfgs = mapAttrsToList configForNetwork cfg;
collectAllNetworkAttrs = x: concatAttrs (map (y: y.${x}) networkCfgs);
in {
assertions = concatMap (wgName: let
inherit
(extraLib.wireguard wgName)
externalPeerNamesRaw
usedAddresses
associatedNodes
;
wgCfg = cfg.${wgName};
wgCfgOf = node: nodes.${node}.config.extra.wireguard.${wgName};
duplicatePeers = duplicates externalPeerNamesRaw;
duplicateAddrs = duplicates (map (x: head (splitString "/" x)) usedAddresses);
in [
{
assertion = any (n: nodes.${n}.config.extra.wireguard.${wgName}.server.enable) associatedNodes;
message = "Wireguard network '${wgName}': At least one node must be a server.";
}
{
assertion = duplicatePeers == [];
message = "Wireguard network '${wgName}': Multiple definitions for external peer(s):${concatMapStrings (x: " '${x}'") duplicatePeers}";
}
{
assertion = duplicateAddrs == [];
message = "Wireguard network '${wgName}': Addresses used multiple times: ${concatStringsSep ", " duplicateAddrs}";
}
{
assertion = wgCfg.server.externalPeers != {} -> wgCfg.server.enable;
message = "Wireguard network '${wgName}': Defining external peers requires server.enable = true.";
}
{
assertion = wgCfg.server.enable == (wgCfg.via == null);
message = "Wireguard network '${wgName}': A via server must be defined exactly iff this isn't a server node.";
}
{
assertion = wgCfg.via != null -> (wgCfgOf wgCfg.via).server.enable;
message = "Wireguard network '${wgName}': The specified via node '${wgCfg.via}' must be a wireguard server.";
}
# TODO externalPeers != {} -> ip forwarding
# TODO no overlapping allowed ip range? 0.0.0.0 would be ok to overlap though
]) (attrNames cfg);
networking.firewall.allowedUDPPorts = mkIf (cfg.server.enable && cfg.server.openFirewall) [cfg.server.port];
rekey.secrets = collectAllNetworkAttrs "secrets";
systemd.network = {
netdevs = collectAllNetworkAttrs "netdevs";
networks = collectAllNetworkAttrs "networks";
};
});
config = mkIf (cfg != {}) (mergeToplevelConfigs
["assertions" "rekey" "networking" "systemd"]
(mapAttrsToList configForNetwork cfg));
}