1
1
Fork 1
mirror of https://github.com/oddlama/nixos-extra-modules.git synced 2025-10-10 22:00:39 +02:00

feat: allow multiple interfaces in guests

This commit is contained in:
Patrick 2024-12-19 21:46:11 +01:00
parent da6945497b
commit 6a4736e077
No known key found for this signature in database
GPG key ID: 451F95EFB8BECD0F
4 changed files with 330 additions and 245 deletions

View file

@ -1,5 +1,11 @@
_guestName: guestCfg: {lib, ...}: let
inherit (lib) mkForce;
inherit
(lib)
mkForce
nameValuePair
listToAttrs
flip
;
in {
node.name = guestCfg.nodeName;
node.type = guestCfg.backend;
@ -11,18 +17,23 @@ in {
};
documentation.enable = mkForce false;
systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = {
matchConfig.Name = guestCfg.networking.mainLinkName;
DHCP = "yes";
# XXX: Do we really want this?
dhcpV4Config.UseDNS = false;
dhcpV6Config.UseDNS = false;
ipv6AcceptRAConfig.UseDNS = false;
networkConfig = {
IPv6PrivacyExtensions = "yes";
MulticastDNS = true;
IPv6AcceptRA = true;
};
linkConfig.RequiredForOnline = "routable";
};
systemd.network.networks = listToAttrs (
flip map guestCfg.networking.links (
name:
nameValuePair "10-${name}" {
matchConfig.Name = name;
DHCP = "yes";
# XXX: Do we really want this?
dhcpV4Config.UseDNS = false;
dhcpV6Config.UseDNS = false;
ipv6AcceptRAConfig.UseDNS = false;
networkConfig = {
IPv6PrivacyExtensions = "yes";
MulticastDNS = true;
IPv6AcceptRA = true;
};
linkConfig.RequiredForOnline = "routable";
}
)
);
}

View file

@ -12,10 +12,10 @@ guestName: guestCfg: {
nameValuePair
;
in {
inherit (guestCfg.container) macvlans;
ephemeral = true;
privateNetwork = true;
autoStart = guestCfg.autostart;
macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
extraFlags = [
"--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}"
];
@ -28,7 +28,11 @@ in {
);
nixosConfiguration = (import "${inputs.nixpkgs}/nixos/lib/eval-config.nix") {
specialArgs = guestCfg.extraSpecialArgs;
prefix = ["nodes" "${config.node.name}-${guestName}" "config"];
prefix = [
"nodes"
"${config.node.name}-${guestName}"
"config"
];
system = null;
modules =
[
@ -49,13 +53,15 @@ in {
# and not recursive. This allows us to have a fileSystems entry for each
# bindMount which other stuff can depend upon (impermanence adds dependencies
# to the state fs).
fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg:
nameValuePair zfsCfg.guestMountpoint {
neededForBoot = true;
fsType = "none";
device = zfsCfg.guestMountpoint;
options = ["bind"];
});
fileSystems = flip mapAttrs' guestCfg.zfs (
_: zfsCfg:
nameValuePair zfsCfg.guestMountpoint {
neededForBoot = true;
fsType = "none";
device = zfsCfg.guestMountpoint;
options = ["bind"];
}
);
}
(import ./common-guest-config.nix guestName guestCfg)
]

View file

@ -10,11 +10,15 @@
attrNames
attrValues
attrsToList
length
splitString
elemAt
disko
escapeShellArg
flatten
flip
foldl'
forEach
groupBy
hasInfix
hasPrefix
@ -35,51 +39,57 @@
;
# All available backends
backends = ["microvm" "container"];
backends = [
"microvm"
"container"
];
guestsByBackend =
lib.genAttrs backends (_: {})
// mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests));
# List the necessary mount units for the given guest
fsMountUnitsFor = guestCfg:
map
(x: "${utils.escapeSystemdPath x.hostMountpoint}.mount")
(attrValues guestCfg.zfs);
fsMountUnitsFor = guestCfg: map (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") (attrValues guestCfg.zfs);
# Configuration required on the host for a specific guest
defineGuest = _guestName: guestCfg: {
# Add the required datasets to the disko configuration of the machine
disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: {
${zfsCfg.pool}.datasets.${zfsCfg.dataset} =
# We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests
disko.zfs.unmountable;
}));
disko.devices.zpool = mkMerge (
flip map (attrValues guestCfg.zfs) (zfsCfg: {
${zfsCfg.pool}.datasets.${zfsCfg.dataset} =
# We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests
disko.zfs.unmountable;
})
);
# Ensure that the zfs dataset exists before it is mounted.
systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let
fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
in {
"zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = {
wantedBy = [fsMountUnit];
before = [fsMountUnit];
after = [
"zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service"
"zfs-mount.target"
];
unitConfig.DefaultDependencies = "no";
serviceConfig.Type = "oneshot";
script = let
poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}";
diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset};
in ''
export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
${diskoDataset._create}
fi
'';
};
}));
systemd.services = mkMerge (
flip map (attrValues guestCfg.zfs) (
zfsCfg: let
fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
in {
"zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = {
wantedBy = [fsMountUnit];
before = [fsMountUnit];
after = [
"zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service"
"zfs-mount.target"
];
unitConfig.DefaultDependencies = "no";
serviceConfig.Type = "oneshot";
script = let
poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}";
diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset};
in ''
export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
${diskoDataset._create}
fi
'';
};
}
)
);
};
defineMicrovm = guestName: guestCfg: {
@ -123,159 +133,188 @@ in {
};
options.containers = mkOption {
type = types.attrsOf (types.submodule (submod: {
options.nixosConfiguration = mkOption {
type = types.unspecified;
default = null;
description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
};
config = mkIf (submod.config.nixosConfiguration != null) ({
path = submod.config.nixosConfiguration.config.system.build.toplevel;
}
// optionalAttrs (config ? topology) {
_nix_topology_config = submod.config.nixosConfiguration.config;
});
}));
type = types.attrsOf (
types.submodule (submod: {
options.nixosConfiguration = mkOption {
type = types.unspecified;
default = null;
description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
};
config = mkIf (submod.config.nixosConfiguration != null) (
{
path = submod.config.nixosConfiguration.config.system.build.toplevel;
}
// optionalAttrs (config ? topology) {
_nix_topology_config = submod.config.nixosConfiguration.config;
}
);
})
);
};
options.guests = mkOption {
default = {};
description = "Defines the actual vms and handles the necessary base setup for them.";
type = types.attrsOf (types.submodule (submod: {
options = {
nodeName = mkOption {
type = types.str;
default = "${config.node.name}-${submod.config._module.args.name}";
description = ''
The name of the resulting node. By default this will be a compound name
of the host's name and the guest's name to avoid name clashes. Can be
overwritten to designate special names to specific guests.
'';
};
backend = mkOption {
type = types.enum backends;
description = ''
Determines how the guest will be hosted. You can currently choose
between microvm based deployment, or nixos containers.
'';
};
extraSpecialArgs = mkOption {
type = types.attrs;
default = {};
example = literalExpression "{ inherit inputs; }";
description = ''
Extra `specialArgs` passed to each guest system definition. This
option can be used to pass additional arguments to all modules.
'';
};
# Options for the microvm backend
microvm = {
system = mkOption {
type = types.attrsOf (
types.submodule (submod: {
options = {
nodeName = mkOption {
type = types.str;
description = "The system that this microvm should use";
default = "${config.node.name}-${submod.config._module.args.name}";
description = ''
The name of the resulting node. By default this will be a compound name
of the host's name and the guest's name to avoid name clashes. Can be
overwritten to designate special names to specific guests.
'';
};
macvtap = mkOption {
type = types.str;
description = "The host interface to which the microvm should be attached via macvtap";
backend = mkOption {
type = types.enum backends;
description = ''
Determines how the guest will be hosted. You can currently choose
between microvm based deployment, or nixos containers.
'';
};
baseMac = mkOption {
type = types.net.mac;
description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
default = "02:01:27:00:00:00";
extraSpecialArgs = mkOption {
type = types.attrs;
default = {};
example = literalExpression "{ inherit inputs; }";
description = ''
Extra `specialArgs` passed to each guest system definition. This
option can be used to pass additional arguments to all modules.
'';
};
mac = mkOption {
type = types.net.mac;
description = "The MAC address for the guest's macvtap interface";
default = let
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
in
(net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name};
};
};
# Options for the container backend
container = {
macvlan = mkOption {
type = types.str;
description = "The host interface to which the container should be attached";
};
};
networking.mainLinkName = mkOption {
type = types.str;
description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host.";
default =
if submod.config.backend == "microvm"
then submod.config.microvm.macvtap
else if submod.config.backend == "container"
then "mv-${submod.config.container.macvlan}"
else throw "Invalid backend";
};
zfs = mkOption {
description = "zfs datasets to mount into the guest";
default = {};
type = types.attrsOf (types.submodule (zfsSubmod: {
options = {
pool = mkOption {
type = types.str;
description = "The host's zfs pool on which the dataset resides";
};
dataset = mkOption {
type = types.str;
example = "safe/guests/mycontainer";
description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
};
hostMountpoint = mkOption {
type = types.path;
default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}";
example = "/guests/mycontainer/persist";
description = "The host's mountpoint for the guest's dataset";
};
guestMountpoint = mkOption {
type = types.path;
default = zfsSubmod.config._module.args.name;
example = "/persist";
description = "The mountpoint inside the guest.";
};
# Options for the microvm backend
microvm = {
system = mkOption {
type = types.str;
description = "The system that this microvm should use";
};
}));
};
autostart = mkOption {
type = types.bool;
default = false;
description = "Whether this guest should be started automatically with the host";
};
baseMac = mkOption {
type = types.net.mac;
description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
default = "02:01:27:00:00:00";
};
interfaces = mkOption {
description = "An attrset correlating the host interface to which the microvm should be attached via macvtap, with its mac address";
type = types.attrsOf (
types.submodule (submod-iface: {
options.mac = mkOption {
type = types.net.mac;
description = "The MAC address for the guest's macvtap interface";
default = let
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
in
(net.mac.assignMacs base 24 [] (
flatten (
flip mapAttrsToList config.guests (
name: value: forEach (attrNames value.microvm.interfaces) (iface: "${name}-${iface}")
)
)
))
."${submod.config._module.args.name}-${submod-iface.config._module.args.name}";
};
})
);
default = {};
};
};
modules = mkOption {
type = types.listOf types.unspecified;
default = [];
description = "Additional modules to load";
# Options for the container backend
container = {
macvlans = mkOption {
type = types.listOf types.str;
description = ''
The macvlans to be created for the container.
Can be either an interface name in which case the container interface will be called mv-<name> or a pair
of <host iface name>:<container iface name>.
'';
};
};
networking.links = mkOption {
type = types.listOf types.str;
description = "The ethernet links inside of the guest. For containers, these cannot be named similar to an existing interface on the host.";
default =
if submod.config.backend == "microvm"
then (flip mapAttrsToList submod.config.microvm.interfaces (name: _: name))
else if submod.config.backend == "container"
then
(forEach submod.config.container.macvlans (
name: let
split = splitString ":" name;
in
if length split > 1
then elemAt split 1
else "mv-${name}"
))
else throw "Invalid backend";
};
zfs = mkOption {
description = "zfs datasets to mount into the guest";
default = {};
type = types.attrsOf (
types.submodule (zfsSubmod: {
options = {
pool = mkOption {
type = types.str;
description = "The host's zfs pool on which the dataset resides";
};
dataset = mkOption {
type = types.str;
example = "safe/guests/mycontainer";
description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
};
hostMountpoint = mkOption {
type = types.path;
default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}";
example = "/guests/mycontainer/persist";
description = "The host's mountpoint for the guest's dataset";
};
guestMountpoint = mkOption {
type = types.path;
default = zfsSubmod.config._module.args.name;
example = "/persist";
description = "The mountpoint inside the guest.";
};
};
})
);
};
autostart = mkOption {
type = types.bool;
default = false;
description = "Whether this guest should be started automatically with the host";
};
modules = mkOption {
type = types.listOf types.unspecified;
default = [];
description = "Additional modules to load";
};
};
};
}));
})
);
};
config = mkIf (config.guests != {}) (
mkMerge [
{
systemd.tmpfiles.rules = [
"d /guests 0700 root root -"
];
config = mkIf (config.guests != {}) (mkMerge [
{
systemd.tmpfiles.rules = [
"d /guests 0700 root root -"
];
# To enable shared folders we need to do all fileSystems entries ourselfs
fileSystems = let
zfsDefs = flatten (flip mapAttrsToList config.guests (
# To enable shared folders we need to do all fileSystems entries ourselfs
fileSystems = let
zfsDefs = flatten (
flip mapAttrsToList config.guests (
_: guestCfg:
flip mapAttrsToList guestCfg.zfs (
_: zfsCfg: {
@ -283,39 +322,53 @@ in {
inherit (zfsCfg) hostMountpoint;
}
)
));
# Due to limitations in zfs mounting we need to explicitly set an order in which
# any dataset gets mounted
zfsDefsByPath = flip groupBy zfsDefs (x: x.path);
in
mkMerge (flip mapAttrsToList zfsDefsByPath (_: defs:
(foldl' ({
prev,
res,
}: elem: {
prev = elem;
res =
res
// {
${elem.hostMountpoint} = {
fsType = "zfs";
options =
["zfsutil"]
++ optional (prev != null) "x-systemd.requires-mounts-for=${warnIf
(hasInfix " " prev.hostMountpoint) "HostMountpoint ${prev.hostMountpoint} cannot contain a space"
prev.hostMountpoint}";
device = elem.path;
};
};
})
{
prev = null;
res = {};
}
defs)
.res));
)
);
# Due to limitations in zfs mounting we need to explicitly set an order in which
# any dataset gets mounted
zfsDefsByPath = flip groupBy zfsDefs (x: x.path);
in
mkMerge (
flip mapAttrsToList zfsDefsByPath (
_: defs:
(
foldl'
(
{
prev,
res,
}: elem: {
prev = elem;
res =
res
// {
${elem.hostMountpoint} = {
fsType = "zfs";
options =
["zfsutil"]
++ optional (prev != null)
"x-systemd.requires-mounts-for=${
warnIf (hasInfix " " prev.hostMountpoint)
"HostMountpoint ${prev.hostMountpoint} cannot contain a space"
prev.hostMountpoint
}";
device = elem.path;
};
};
}
)
{
prev = null;
res = {};
}
defs
)
.res
)
);
assertions = flatten (flip mapAttrsToList config.guests (
assertions = flatten (
flip mapAttrsToList config.guests (
guestName: guestCfg:
flip mapAttrsToList guestCfg.zfs (
zfsName: zfsCfg: {
@ -323,11 +376,17 @@ in {
message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path.";
}
)
));
}
(mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (mapAttrsToList defineGuest config.guests))
(mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container))
(mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm))
]
);
)
);
}
(mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (
mapAttrsToList defineGuest config.guests
))
(mergeToplevelConfigs ["containers" "systemd"] (
mapAttrsToList defineContainer guestsByBackend.container
))
(mergeToplevelConfigs ["microvm" "systemd"] (
mapAttrsToList defineMicrovm guestsByBackend.microvm
))
]);
}

View file

@ -5,10 +5,13 @@ guestName: guestCfg: {
}: let
inherit
(lib)
concatMapAttrs
flip
mapAttrs
mapAttrsToList
mkDefault
mkForce
replaceStrings
;
in {
specialArgs = guestCfg.extraSpecialArgs;
@ -19,13 +22,15 @@ in {
guestCfg.modules
++ [
(import ./common-guest-config.nix guestName guestCfg)
({config, ...}: {
# Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"];
})
(
{config, ...}: {
# Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"];
}
)
];
lib.microvm.mac = guestCfg.microvm.mac;
lib.microvm.interfaces = guestCfg.microvm.interfaces;
microvm = {
hypervisor = mkDefault "qemu";
@ -41,17 +46,17 @@ in {
writableStoreOverlay = "/nix/.rw-store";
# MACVTAP bridge to the host's network
interfaces = [
{
interfaces = flip mapAttrsToList guestCfg.microvm.interfaces (
interface: {mac, ...}: {
type = "macvtap";
id = "vm-${guestName}";
inherit (guestCfg.microvm) mac;
id = "vm-${replaceStrings [":"] [""] mac}";
inherit mac;
macvtap = {
link = guestCfg.microvm.macvtap;
link = interface;
mode = "bridge";
};
}
];
);
shares =
[
@ -73,9 +78,13 @@ in {
);
};
networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac;
systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce {
MACAddress = guestCfg.microvm.mac;
};
networking.renameInterfacesByMac = flip mapAttrs guestCfg.microvm.interfaces (_: {mac, ...}: mac);
systemd.network.networks = flip concatMapAttrs guestCfg.microvm.interfaces (
name: {mac, ...}: {
"10-${name}".matchConfig = mkForce {
MACAddress = mac;
};
}
);
};
}