mirror of
https://github.com/oddlama/nixos-extra-modules.git
synced 2025-10-10 22:00:39 +02:00
feat: allow multiple interfaces in guests
This commit is contained in:
Patrick
parent
da6945497b
commit
6a4736e077
4 changed files with 330 additions and 245 deletions
|
|
@ -1,5 +1,11 @@
|
|||
_guestName: guestCfg: {lib, ...}: let
|
||||
inherit (lib) mkForce;
|
||||
inherit
|
||||
(lib)
|
||||
mkForce
|
||||
nameValuePair
|
||||
listToAttrs
|
||||
flip
|
||||
;
|
||||
in {
|
||||
node.name = guestCfg.nodeName;
|
||||
node.type = guestCfg.backend;
|
||||
|
|
@ -11,8 +17,11 @@ in {
|
|||
};
|
||||
documentation.enable = mkForce false;
|
||||
|
||||
systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = {
|
||||
matchConfig.Name = guestCfg.networking.mainLinkName;
|
||||
systemd.network.networks = listToAttrs (
|
||||
flip map guestCfg.networking.links (
|
||||
name:
|
||||
nameValuePair "10-${name}" {
|
||||
matchConfig.Name = name;
|
||||
DHCP = "yes";
|
||||
# XXX: Do we really want this?
|
||||
dhcpV4Config.UseDNS = false;
|
||||
|
|
@ -24,5 +33,7 @@ in {
|
|||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ guestName: guestCfg: {
|
|||
nameValuePair
|
||||
;
|
||||
in {
|
||||
inherit (guestCfg.container) macvlans;
|
||||
ephemeral = true;
|
||||
privateNetwork = true;
|
||||
autoStart = guestCfg.autostart;
|
||||
macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
|
||||
extraFlags = [
|
||||
"--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}"
|
||||
];
|
||||
|
|
@ -28,7 +28,11 @@ in {
|
|||
);
|
||||
nixosConfiguration = (import "${inputs.nixpkgs}/nixos/lib/eval-config.nix") {
|
||||
specialArgs = guestCfg.extraSpecialArgs;
|
||||
prefix = ["nodes" "${config.node.name}-${guestName}" "config"];
|
||||
prefix = [
|
||||
"nodes"
|
||||
"${config.node.name}-${guestName}"
|
||||
"config"
|
||||
];
|
||||
system = null;
|
||||
modules =
|
||||
[
|
||||
|
|
@ -49,13 +53,15 @@ in {
|
|||
# and not recursive. This allows us to have a fileSystems entry for each
|
||||
# bindMount which other stuff can depend upon (impermanence adds dependencies
|
||||
# to the state fs).
|
||||
fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg:
|
||||
fileSystems = flip mapAttrs' guestCfg.zfs (
|
||||
_: zfsCfg:
|
||||
nameValuePair zfsCfg.guestMountpoint {
|
||||
neededForBoot = true;
|
||||
fsType = "none";
|
||||
device = zfsCfg.guestMountpoint;
|
||||
options = ["bind"];
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
(import ./common-guest-config.nix guestName guestCfg)
|
||||
]
|
||||
|
|
|
|||
|
|
@ -10,11 +10,15 @@
|
|||
attrNames
|
||||
attrValues
|
||||
attrsToList
|
||||
length
|
||||
splitString
|
||||
elemAt
|
||||
disko
|
||||
escapeShellArg
|
||||
flatten
|
||||
flip
|
||||
foldl'
|
||||
forEach
|
||||
groupBy
|
||||
hasInfix
|
||||
hasPrefix
|
||||
|
|
@ -35,29 +39,33 @@
|
|||
;
|
||||
|
||||
# All available backends
|
||||
backends = ["microvm" "container"];
|
||||
backends = [
|
||||
"microvm"
|
||||
"container"
|
||||
];
|
||||
|
||||
guestsByBackend =
|
||||
lib.genAttrs backends (_: {})
|
||||
// mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests));
|
||||
|
||||
# List the necessary mount units for the given guest
|
||||
fsMountUnitsFor = guestCfg:
|
||||
map
|
||||
(x: "${utils.escapeSystemdPath x.hostMountpoint}.mount")
|
||||
(attrValues guestCfg.zfs);
|
||||
fsMountUnitsFor = guestCfg: map (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") (attrValues guestCfg.zfs);
|
||||
|
||||
# Configuration required on the host for a specific guest
|
||||
defineGuest = _guestName: guestCfg: {
|
||||
# Add the required datasets to the disko configuration of the machine
|
||||
disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: {
|
||||
disko.devices.zpool = mkMerge (
|
||||
flip map (attrValues guestCfg.zfs) (zfsCfg: {
|
||||
${zfsCfg.pool}.datasets.${zfsCfg.dataset} =
|
||||
# We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests
|
||||
disko.zfs.unmountable;
|
||||
}));
|
||||
})
|
||||
);
|
||||
|
||||
# Ensure that the zfs dataset exists before it is mounted.
|
||||
systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let
|
||||
systemd.services = mkMerge (
|
||||
flip map (attrValues guestCfg.zfs) (
|
||||
zfsCfg: let
|
||||
fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
|
||||
in {
|
||||
"zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = {
|
||||
|
|
@ -79,7 +87,9 @@
|
|||
fi
|
||||
'';
|
||||
};
|
||||
}));
|
||||
}
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
defineMicrovm = guestName: guestCfg: {
|
||||
|
|
@ -123,25 +133,30 @@ in {
|
|||
};
|
||||
|
||||
options.containers = mkOption {
|
||||
type = types.attrsOf (types.submodule (submod: {
|
||||
type = types.attrsOf (
|
||||
types.submodule (submod: {
|
||||
options.nixosConfiguration = mkOption {
|
||||
type = types.unspecified;
|
||||
default = null;
|
||||
description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
|
||||
};
|
||||
config = mkIf (submod.config.nixosConfiguration != null) ({
|
||||
config = mkIf (submod.config.nixosConfiguration != null) (
|
||||
{
|
||||
path = submod.config.nixosConfiguration.config.system.build.toplevel;
|
||||
}
|
||||
// optionalAttrs (config ? topology) {
|
||||
_nix_topology_config = submod.config.nixosConfiguration.config;
|
||||
});
|
||||
}));
|
||||
}
|
||||
);
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
options.guests = mkOption {
|
||||
default = {};
|
||||
description = "Defines the actual vms and handles the necessary base setup for them.";
|
||||
type = types.attrsOf (types.submodule (submod: {
|
||||
type = types.attrsOf (
|
||||
types.submodule (submod: {
|
||||
options = {
|
||||
nodeName = mkOption {
|
||||
type = types.str;
|
||||
|
|
@ -178,50 +193,72 @@ in {
|
|||
description = "The system that this microvm should use";
|
||||
};
|
||||
|
||||
macvtap = mkOption {
|
||||
type = types.str;
|
||||
description = "The host interface to which the microvm should be attached via macvtap";
|
||||
};
|
||||
|
||||
baseMac = mkOption {
|
||||
type = types.net.mac;
|
||||
description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
|
||||
default = "02:01:27:00:00:00";
|
||||
};
|
||||
|
||||
mac = mkOption {
|
||||
interfaces = mkOption {
|
||||
description = "An attrset correlating the host interface to which the microvm should be attached via macvtap, with its mac address";
|
||||
type = types.attrsOf (
|
||||
types.submodule (submod-iface: {
|
||||
options.mac = mkOption {
|
||||
type = types.net.mac;
|
||||
description = "The MAC address for the guest's macvtap interface";
|
||||
default = let
|
||||
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
|
||||
in
|
||||
(net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name};
|
||||
(net.mac.assignMacs base 24 [] (
|
||||
flatten (
|
||||
flip mapAttrsToList config.guests (
|
||||
name: value: forEach (attrNames value.microvm.interfaces) (iface: "${name}-${iface}")
|
||||
)
|
||||
)
|
||||
))
|
||||
."${submod.config._module.args.name}-${submod-iface.config._module.args.name}";
|
||||
};
|
||||
})
|
||||
);
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
|
||||
# Options for the container backend
|
||||
container = {
|
||||
macvlan = mkOption {
|
||||
type = types.str;
|
||||
description = "The host interface to which the container should be attached";
|
||||
macvlans = mkOption {
|
||||
type = types.listOf types.str;
|
||||
description = ''
|
||||
The macvlans to be created for the container.
|
||||
Can be either an interface name in which case the container interface will be called mv-<name> or a pair
|
||||
of <host iface name>:<container iface name>.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
networking.mainLinkName = mkOption {
|
||||
type = types.str;
|
||||
description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host.";
|
||||
networking.links = mkOption {
|
||||
type = types.listOf types.str;
|
||||
description = "The ethernet links inside of the guest. For containers, these cannot be named similar to an existing interface on the host.";
|
||||
default =
|
||||
if submod.config.backend == "microvm"
|
||||
then submod.config.microvm.macvtap
|
||||
then (flip mapAttrsToList submod.config.microvm.interfaces (name: _: name))
|
||||
else if submod.config.backend == "container"
|
||||
then "mv-${submod.config.container.macvlan}"
|
||||
then
|
||||
(forEach submod.config.container.macvlans (
|
||||
name: let
|
||||
split = splitString ":" name;
|
||||
in
|
||||
if length split > 1
|
||||
then elemAt split 1
|
||||
else "mv-${name}"
|
||||
))
|
||||
else throw "Invalid backend";
|
||||
};
|
||||
|
||||
zfs = mkOption {
|
||||
description = "zfs datasets to mount into the guest";
|
||||
default = {};
|
||||
type = types.attrsOf (types.submodule (zfsSubmod: {
|
||||
type = types.attrsOf (
|
||||
types.submodule (zfsSubmod: {
|
||||
options = {
|
||||
pool = mkOption {
|
||||
type = types.str;
|
||||
|
|
@ -248,7 +285,8 @@ in {
|
|||
description = "The mountpoint inside the guest.";
|
||||
};
|
||||
};
|
||||
}));
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
autostart = mkOption {
|
||||
|
|
@ -263,11 +301,11 @@ in {
|
|||
description = "Additional modules to load";
|
||||
};
|
||||
};
|
||||
}));
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
config = mkIf (config.guests != {}) (
|
||||
mkMerge [
|
||||
config = mkIf (config.guests != {}) (mkMerge [
|
||||
{
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /guests 0700 root root -"
|
||||
|
|
@ -275,7 +313,8 @@ in {
|
|||
|
||||
# To enable shared folders we need to do all fileSystems entries ourselfs
|
||||
fileSystems = let
|
||||
zfsDefs = flatten (flip mapAttrsToList config.guests (
|
||||
zfsDefs = flatten (
|
||||
flip mapAttrsToList config.guests (
|
||||
_: guestCfg:
|
||||
flip mapAttrsToList guestCfg.zfs (
|
||||
_: zfsCfg: {
|
||||
|
|
@ -283,13 +322,19 @@ in {
|
|||
inherit (zfsCfg) hostMountpoint;
|
||||
}
|
||||
)
|
||||
));
|
||||
)
|
||||
);
|
||||
# Due to limitations in zfs mounting we need to explicitly set an order in which
|
||||
# any dataset gets mounted
|
||||
zfsDefsByPath = flip groupBy zfsDefs (x: x.path);
|
||||
in
|
||||
mkMerge (flip mapAttrsToList zfsDefsByPath (_: defs:
|
||||
(foldl' ({
|
||||
mkMerge (
|
||||
flip mapAttrsToList zfsDefsByPath (
|
||||
_: defs:
|
||||
(
|
||||
foldl'
|
||||
(
|
||||
{
|
||||
prev,
|
||||
res,
|
||||
}: elem: {
|
||||
|
|
@ -301,21 +346,29 @@ in {
|
|||
fsType = "zfs";
|
||||
options =
|
||||
["zfsutil"]
|
||||
++ optional (prev != null) "x-systemd.requires-mounts-for=${warnIf
|
||||
(hasInfix " " prev.hostMountpoint) "HostMountpoint ${prev.hostMountpoint} cannot contain a space"
|
||||
prev.hostMountpoint}";
|
||||
++ optional (prev != null)
|
||||
"x-systemd.requires-mounts-for=${
|
||||
warnIf (hasInfix " " prev.hostMountpoint)
|
||||
"HostMountpoint ${prev.hostMountpoint} cannot contain a space"
|
||||
prev.hostMountpoint
|
||||
}";
|
||||
device = elem.path;
|
||||
};
|
||||
};
|
||||
})
|
||||
}
|
||||
)
|
||||
{
|
||||
prev = null;
|
||||
res = {};
|
||||
}
|
||||
defs)
|
||||
.res));
|
||||
defs
|
||||
)
|
||||
.res
|
||||
)
|
||||
);
|
||||
|
||||
assertions = flatten (flip mapAttrsToList config.guests (
|
||||
assertions = flatten (
|
||||
flip mapAttrsToList config.guests (
|
||||
guestName: guestCfg:
|
||||
flip mapAttrsToList guestCfg.zfs (
|
||||
zfsName: zfsCfg: {
|
||||
|
|
@ -323,11 +376,17 @@ in {
|
|||
message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path.";
|
||||
}
|
||||
)
|
||||
));
|
||||
}
|
||||
(mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (mapAttrsToList defineGuest config.guests))
|
||||
(mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container))
|
||||
(mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm))
|
||||
]
|
||||
)
|
||||
);
|
||||
}
|
||||
(mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (
|
||||
mapAttrsToList defineGuest config.guests
|
||||
))
|
||||
(mergeToplevelConfigs ["containers" "systemd"] (
|
||||
mapAttrsToList defineContainer guestsByBackend.container
|
||||
))
|
||||
(mergeToplevelConfigs ["microvm" "systemd"] (
|
||||
mapAttrsToList defineMicrovm guestsByBackend.microvm
|
||||
))
|
||||
]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@ guestName: guestCfg: {
|
|||
}: let
|
||||
inherit
|
||||
(lib)
|
||||
concatMapAttrs
|
||||
flip
|
||||
mapAttrs
|
||||
mapAttrsToList
|
||||
mkDefault
|
||||
mkForce
|
||||
replaceStrings
|
||||
;
|
||||
in {
|
||||
specialArgs = guestCfg.extraSpecialArgs;
|
||||
|
|
@ -19,13 +22,15 @@ in {
|
|||
guestCfg.modules
|
||||
++ [
|
||||
(import ./common-guest-config.nix guestName guestCfg)
|
||||
({config, ...}: {
|
||||
(
|
||||
{config, ...}: {
|
||||
# Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
|
||||
boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"];
|
||||
})
|
||||
}
|
||||
)
|
||||
];
|
||||
|
||||
lib.microvm.mac = guestCfg.microvm.mac;
|
||||
lib.microvm.interfaces = guestCfg.microvm.interfaces;
|
||||
|
||||
microvm = {
|
||||
hypervisor = mkDefault "qemu";
|
||||
|
|
@ -41,17 +46,17 @@ in {
|
|||
writableStoreOverlay = "/nix/.rw-store";
|
||||
|
||||
# MACVTAP bridge to the host's network
|
||||
interfaces = [
|
||||
{
|
||||
interfaces = flip mapAttrsToList guestCfg.microvm.interfaces (
|
||||
interface: {mac, ...}: {
|
||||
type = "macvtap";
|
||||
id = "vm-${guestName}";
|
||||
inherit (guestCfg.microvm) mac;
|
||||
id = "vm-${replaceStrings [":"] [""] mac}";
|
||||
inherit mac;
|
||||
macvtap = {
|
||||
link = guestCfg.microvm.macvtap;
|
||||
link = interface;
|
||||
mode = "bridge";
|
||||
};
|
||||
}
|
||||
];
|
||||
);
|
||||
|
||||
shares =
|
||||
[
|
||||
|
|
@ -73,9 +78,13 @@ in {
|
|||
);
|
||||
};
|
||||
|
||||
networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac;
|
||||
systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce {
|
||||
MACAddress = guestCfg.microvm.mac;
|
||||
};
|
||||
networking.renameInterfacesByMac = flip mapAttrs guestCfg.microvm.interfaces (_: {mac, ...}: mac);
|
||||
systemd.network.networks = flip concatMapAttrs guestCfg.microvm.interfaces (
|
||||
name: {mac, ...}: {
|
||||
"10-${name}".matchConfig = mkForce {
|
||||
MACAddress = mac;
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue