sokai/mynixos-extra-modules
1
0
Fork 0
forked from mirrors_public/oddlama_nixos-extra-modules
Code Pull requests Activity

feat: allow multiple interfaces in guests

Browse source
This commit is contained in:
Patrick 2024-12-19 21:46:11 +01:00
parent da6945497b
commit 6a4736e077
No known key found for this signature in database
GPG key ID: 451F95EFB8BECD0F
4 changed files with 330 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

41
modules/guests/common-guest-config.nix
View file

@ -1,5 +1,11 @@
_guestName: guestCfg: {lib, ...}: let _guestName: guestCfg: {lib, ...}: let
inherit (lib) mkForce; inherit
(lib)
mkForce
nameValuePair
listToAttrs
flip
;
in { in {
node.name = guestCfg.nodeName; node.name = guestCfg.nodeName;
node.type = guestCfg.backend; node.type = guestCfg.backend;
@ -11,18 +17,23 @@ in {
}; };
documentation.enable = mkForce false; documentation.enable = mkForce false;
systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = { systemd.network.networks = listToAttrs (
matchConfig.Name = guestCfg.networking.mainLinkName; flip map guestCfg.networking.links (
DHCP = "yes"; name:
# XXX: Do we really want this? nameValuePair "10-${name}" {
dhcpV4Config.UseDNS = false; matchConfig.Name = name;
dhcpV6Config.UseDNS = false; DHCP = "yes";
ipv6AcceptRAConfig.UseDNS = false; # XXX: Do we really want this?
networkConfig = { dhcpV4Config.UseDNS = false;
IPv6PrivacyExtensions = "yes"; dhcpV6Config.UseDNS = false;
MulticastDNS = true; ipv6AcceptRAConfig.UseDNS = false;
IPv6AcceptRA = true; networkConfig = {
}; IPv6PrivacyExtensions = "yes";
linkConfig.RequiredForOnline = "routable"; MulticastDNS = true;
}; IPv6AcceptRA = true;
};
linkConfig.RequiredForOnline = "routable";
}
)
);
} }

24
modules/guests/container.nix
View file

@ -12,10 +12,10 @@ guestName: guestCfg: {
nameValuePair nameValuePair
; ;
in { in {
inherit (guestCfg.container) macvlans;
ephemeral = true; ephemeral = true;
privateNetwork = true; privateNetwork = true;
autoStart = guestCfg.autostart; autoStart = guestCfg.autostart;
macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
extraFlags = [ extraFlags = [
"--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}" "--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}"
]; ];
@ -28,7 +28,11 @@ in {
); );
nixosConfiguration = (import "${inputs.nixpkgs}/nixos/lib/eval-config.nix") { nixosConfiguration = (import "${inputs.nixpkgs}/nixos/lib/eval-config.nix") {
specialArgs = guestCfg.extraSpecialArgs; specialArgs = guestCfg.extraSpecialArgs;
prefix = ["nodes" "${config.node.name}-${guestName}" "config"]; prefix = [
"nodes"
"${config.node.name}-${guestName}"
"config"
];
system = null; system = null;
modules = modules =
[ [
@ -49,13 +53,15 @@ in {
# and not recursive. This allows us to have a fileSystems entry for each # and not recursive. This allows us to have a fileSystems entry for each
# bindMount which other stuff can depend upon (impermanence adds dependencies # bindMount which other stuff can depend upon (impermanence adds dependencies
# to the state fs). # to the state fs).
fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg: fileSystems = flip mapAttrs' guestCfg.zfs (
nameValuePair zfsCfg.guestMountpoint { _: zfsCfg:
neededForBoot = true; nameValuePair zfsCfg.guestMountpoint {
fsType = "none"; neededForBoot = true;
device = zfsCfg.guestMountpoint; fsType = "none";
options = ["bind"]; device = zfsCfg.guestMountpoint;
}); options = ["bind"];
}
);
} }
(import ./common-guest-config.nix guestName guestCfg) (import ./common-guest-config.nix guestName guestCfg)
] ]

471
modules/guests/default.nix
View file

@ -10,11 +10,15 @@
attrNames attrNames
attrValues attrValues
attrsToList attrsToList
length
splitString
elemAt
disko disko
escapeShellArg escapeShellArg
flatten flatten
flip flip
foldl' foldl'
forEach
groupBy groupBy
hasInfix hasInfix
hasPrefix hasPrefix
@ -35,51 +39,57 @@
; ;
# All available backends # All available backends
backends = ["microvm" "container"]; backends = [
"microvm"
"container"
];
guestsByBackend = guestsByBackend =
lib.genAttrs backends (_: {}) lib.genAttrs backends (_: {})
// mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests)); // mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests));
# List the necessary mount units for the given guest # List the necessary mount units for the given guest
fsMountUnitsFor = guestCfg: fsMountUnitsFor = guestCfg: map (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") (attrValues guestCfg.zfs);
map
(x: "${utils.escapeSystemdPath x.hostMountpoint}.mount")
(attrValues guestCfg.zfs);
# Configuration required on the host for a specific guest # Configuration required on the host for a specific guest
defineGuest = _guestName: guestCfg: { defineGuest = _guestName: guestCfg: {
# Add the required datasets to the disko configuration of the machine # Add the required datasets to the disko configuration of the machine
disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: { disko.devices.zpool = mkMerge (
${zfsCfg.pool}.datasets.${zfsCfg.dataset} = flip map (attrValues guestCfg.zfs) (zfsCfg: {
# We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests ${zfsCfg.pool}.datasets.${zfsCfg.dataset} =
disko.zfs.unmountable; # We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests
})); disko.zfs.unmountable;
})
);
# Ensure that the zfs dataset exists before it is mounted. # Ensure that the zfs dataset exists before it is mounted.
systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let systemd.services = mkMerge (
fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount"; flip map (attrValues guestCfg.zfs) (
in { zfsCfg: let
"zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = { fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
wantedBy = [fsMountUnit]; in {
before = [fsMountUnit]; "zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = {
after = [ wantedBy = [fsMountUnit];
"zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service" before = [fsMountUnit];
"zfs-mount.target" after = [
]; "zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service"
unitConfig.DefaultDependencies = "no"; "zfs-mount.target"
serviceConfig.Type = "oneshot"; ];
script = let unitConfig.DefaultDependencies = "no";
poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}"; serviceConfig.Type = "oneshot";
diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset}; script = let
in '' poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}";
export PATH=${makeBinPath [pkgs.zfs]}":$PATH" diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset};
if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then in ''
${diskoDataset._create} export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
fi if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
''; ${diskoDataset._create}
}; fi
})); '';
};
}
)
);
}; };
defineMicrovm = guestName: guestCfg: { defineMicrovm = guestName: guestCfg: {
@ -123,159 +133,188 @@ in {
}; };
options.containers = mkOption { options.containers = mkOption {
type = types.attrsOf (types.submodule (submod: { type = types.attrsOf (
options.nixosConfiguration = mkOption { types.submodule (submod: {
type = types.unspecified; options.nixosConfiguration = mkOption {
default = null; type = types.unspecified;
description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you."; default = null;
}; description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
config = mkIf (submod.config.nixosConfiguration != null) ({ };
path = submod.config.nixosConfiguration.config.system.build.toplevel; config = mkIf (submod.config.nixosConfiguration != null) (
} {
// optionalAttrs (config ? topology) { path = submod.config.nixosConfiguration.config.system.build.toplevel;
_nix_topology_config = submod.config.nixosConfiguration.config; }
}); // optionalAttrs (config ? topology) {
})); _nix_topology_config = submod.config.nixosConfiguration.config;
}
);
})
);
}; };
options.guests = mkOption { options.guests = mkOption {
default = {}; default = {};
description = "Defines the actual vms and handles the necessary base setup for them."; description = "Defines the actual vms and handles the necessary base setup for them.";
type = types.attrsOf (types.submodule (submod: { type = types.attrsOf (
options = { types.submodule (submod: {
nodeName = mkOption { options = {
type = types.str; nodeName = mkOption {
default = "${config.node.name}-${submod.config._module.args.name}";
description = ''
The name of the resulting node. By default this will be a compound name
of the host's name and the guest's name to avoid name clashes. Can be
overwritten to designate special names to specific guests.
'';
};
backend = mkOption {
type = types.enum backends;
description = ''
Determines how the guest will be hosted. You can currently choose
between microvm based deployment, or nixos containers.
'';
};
extraSpecialArgs = mkOption {
type = types.attrs;
default = {};
example = literalExpression "{ inherit inputs; }";
description = ''
Extra `specialArgs` passed to each guest system definition. This
option can be used to pass additional arguments to all modules.
'';
};
# Options for the microvm backend
microvm = {
system = mkOption {
type = types.str; type = types.str;
description = "The system that this microvm should use"; default = "${config.node.name}-${submod.config._module.args.name}";
description = ''
The name of the resulting node. By default this will be a compound name
of the host's name and the guest's name to avoid name clashes. Can be
overwritten to designate special names to specific guests.
'';
}; };
macvtap = mkOption { backend = mkOption {
type = types.str; type = types.enum backends;
description = "The host interface to which the microvm should be attached via macvtap"; description = ''
Determines how the guest will be hosted. You can currently choose
between microvm based deployment, or nixos containers.
'';
}; };
baseMac = mkOption { extraSpecialArgs = mkOption {
type = types.net.mac; type = types.attrs;
description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly."; default = {};
default = "02:01:27:00:00:00"; example = literalExpression "{ inherit inputs; }";
description = ''
Extra `specialArgs` passed to each guest system definition. This
option can be used to pass additional arguments to all modules.
'';
}; };
mac = mkOption { # Options for the microvm backend
type = types.net.mac; microvm = {
description = "The MAC address for the guest's macvtap interface"; system = mkOption {
default = let type = types.str;
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00"; description = "The system that this microvm should use";
in
(net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name};
};
};
# Options for the container backend
container = {
macvlan = mkOption {
type = types.str;
description = "The host interface to which the container should be attached";
};
};
networking.mainLinkName = mkOption {
type = types.str;
description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host.";
default =
if submod.config.backend == "microvm"
then submod.config.microvm.macvtap
else if submod.config.backend == "container"
then "mv-${submod.config.container.macvlan}"
else throw "Invalid backend";
};
zfs = mkOption {
description = "zfs datasets to mount into the guest";
default = {};
type = types.attrsOf (types.submodule (zfsSubmod: {
options = {
pool = mkOption {
type = types.str;
description = "The host's zfs pool on which the dataset resides";
};
dataset = mkOption {
type = types.str;
example = "safe/guests/mycontainer";
description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
};
hostMountpoint = mkOption {
type = types.path;
default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}";
example = "/guests/mycontainer/persist";
description = "The host's mountpoint for the guest's dataset";
};
guestMountpoint = mkOption {
type = types.path;
default = zfsSubmod.config._module.args.name;
example = "/persist";
description = "The mountpoint inside the guest.";
};
}; };
}));
};
autostart = mkOption { baseMac = mkOption {
type = types.bool; type = types.net.mac;
default = false; description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
description = "Whether this guest should be started automatically with the host"; default = "02:01:27:00:00:00";
}; };
interfaces = mkOption {
description = "An attrset correlating the host interface to which the microvm should be attached via macvtap, with its mac address";
type = types.attrsOf (
types.submodule (submod-iface: {
options.mac = mkOption {
type = types.net.mac;
description = "The MAC address for the guest's macvtap interface";
default = let
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
in
(net.mac.assignMacs base 24 [] (
flatten (
flip mapAttrsToList config.guests (
name: value: forEach (attrNames value.microvm.interfaces) (iface: "${name}-${iface}")
)
)
))
."${submod.config._module.args.name}-${submod-iface.config._module.args.name}";
};
})
);
default = {};
};
};
modules = mkOption { # Options for the container backend
type = types.listOf types.unspecified; container = {
default = []; macvlans = mkOption {
description = "Additional modules to load"; type = types.listOf types.str;
description = ''
The macvlans to be created for the container.
Can be either an interface name in which case the container interface will be called mv-<name> or a pair
of <host iface name>:<container iface name>.
'';
};
};
networking.links = mkOption {
type = types.listOf types.str;
description = "The ethernet links inside of the guest. For containers, these cannot be named similar to an existing interface on the host.";
default =
if submod.config.backend == "microvm"
then (flip mapAttrsToList submod.config.microvm.interfaces (name: _: name))
else if submod.config.backend == "container"
then
(forEach submod.config.container.macvlans (
name: let
split = splitString ":" name;
in
if length split > 1
then elemAt split 1
else "mv-${name}"
))
else throw "Invalid backend";
};
zfs = mkOption {
description = "zfs datasets to mount into the guest";
default = {};
type = types.attrsOf (
types.submodule (zfsSubmod: {
options = {
pool = mkOption {
type = types.str;
description = "The host's zfs pool on which the dataset resides";
};
dataset = mkOption {
type = types.str;
example = "safe/guests/mycontainer";
description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
};
hostMountpoint = mkOption {
type = types.path;
default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}";
example = "/guests/mycontainer/persist";
description = "The host's mountpoint for the guest's dataset";
};
guestMountpoint = mkOption {
type = types.path;
default = zfsSubmod.config._module.args.name;
example = "/persist";
description = "The mountpoint inside the guest.";
};
};
})
);
};
autostart = mkOption {
type = types.bool;
default = false;
description = "Whether this guest should be started automatically with the host";
};
modules = mkOption {
type = types.listOf types.unspecified;
default = [];
description = "Additional modules to load";
};
}; };
}; })
})); );
}; };
config = mkIf (config.guests != {}) ( config = mkIf (config.guests != {}) (mkMerge [
mkMerge [ {
{ systemd.tmpfiles.rules = [
systemd.tmpfiles.rules = [ "d /guests 0700 root root -"
"d /guests 0700 root root -" ];
];
# To enable shared folders we need to do all fileSystems entries ourselfs # To enable shared folders we need to do all fileSystems entries ourselfs
fileSystems = let fileSystems = let
zfsDefs = flatten (flip mapAttrsToList config.guests ( zfsDefs = flatten (
flip mapAttrsToList config.guests (
_: guestCfg: _: guestCfg:
flip mapAttrsToList guestCfg.zfs ( flip mapAttrsToList guestCfg.zfs (
_: zfsCfg: { _: zfsCfg: {
@ -283,39 +322,53 @@ in {
inherit (zfsCfg) hostMountpoint; inherit (zfsCfg) hostMountpoint;
} }
) )
)); )
# Due to limitations in zfs mounting we need to explicitly set an order in which );
# any dataset gets mounted # Due to limitations in zfs mounting we need to explicitly set an order in which
zfsDefsByPath = flip groupBy zfsDefs (x: x.path); # any dataset gets mounted
in zfsDefsByPath = flip groupBy zfsDefs (x: x.path);
mkMerge (flip mapAttrsToList zfsDefsByPath (_: defs: in
(foldl' ({ mkMerge (
prev, flip mapAttrsToList zfsDefsByPath (
res, _: defs:
}: elem: { (
prev = elem; foldl'
res = (
res {
// { prev,
${elem.hostMountpoint} = { res,
fsType = "zfs"; }: elem: {
options = prev = elem;
["zfsutil"] res =
++ optional (prev != null) "x-systemd.requires-mounts-for=${warnIf res
(hasInfix " " prev.hostMountpoint) "HostMountpoint ${prev.hostMountpoint} cannot contain a space" // {
prev.hostMountpoint}"; ${elem.hostMountpoint} = {
device = elem.path; fsType = "zfs";
}; options =
}; ["zfsutil"]
}) ++ optional (prev != null)
{ "x-systemd.requires-mounts-for=${
prev = null; warnIf (hasInfix " " prev.hostMountpoint)
res = {}; "HostMountpoint ${prev.hostMountpoint} cannot contain a space"
} prev.hostMountpoint
defs) }";
.res)); device = elem.path;
};
};
}
)
{
prev = null;
res = {};
}
defs
)
.res
)
);
assertions = flatten (flip mapAttrsToList config.guests ( assertions = flatten (
flip mapAttrsToList config.guests (
guestName: guestCfg: guestName: guestCfg:
flip mapAttrsToList guestCfg.zfs ( flip mapAttrsToList guestCfg.zfs (
zfsName: zfsCfg: { zfsName: zfsCfg: {
@ -323,11 +376,17 @@ in {
message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path."; message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path.";
} }
) )
)); )
} );
(mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (mapAttrsToList defineGuest config.guests)) }
(mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container)) (mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (
(mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm)) mapAttrsToList defineGuest config.guests
] ))
); (mergeToplevelConfigs ["containers" "systemd"] (
mapAttrsToList defineContainer guestsByBackend.container
))
(mergeToplevelConfigs ["microvm" "systemd"] (
mapAttrsToList defineMicrovm guestsByBackend.microvm
))
]);
} }

39
modules/guests/microvm.nix
View file

@ -5,10 +5,13 @@ guestName: guestCfg: {
}: let }: let
inherit inherit
(lib) (lib)
concatMapAttrs
flip flip
mapAttrs
mapAttrsToList mapAttrsToList
mkDefault mkDefault
mkForce mkForce
replaceStrings
; ;
in { in {
specialArgs = guestCfg.extraSpecialArgs; specialArgs = guestCfg.extraSpecialArgs;
@ -19,13 +22,15 @@ in {
guestCfg.modules guestCfg.modules
++ [ ++ [
(import ./common-guest-config.nix guestName guestCfg) (import ./common-guest-config.nix guestName guestCfg)
({config, ...}: { (
# Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki {config, ...}: {
boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"]; # Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
}) boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"];
}
)
]; ];
lib.microvm.mac = guestCfg.microvm.mac; lib.microvm.interfaces = guestCfg.microvm.interfaces;
microvm = { microvm = {
hypervisor = mkDefault "qemu"; hypervisor = mkDefault "qemu";
@ -41,17 +46,17 @@ in {
writableStoreOverlay = "/nix/.rw-store"; writableStoreOverlay = "/nix/.rw-store";
# MACVTAP bridge to the host's network # MACVTAP bridge to the host's network
interfaces = [ interfaces = flip mapAttrsToList guestCfg.microvm.interfaces (
{ interface: {mac, ...}: {
type = "macvtap"; type = "macvtap";
id = "vm-${guestName}"; id = "vm-${replaceStrings [":"] [""] mac}";
inherit (guestCfg.microvm) mac; inherit mac;
macvtap = { macvtap = {
link = guestCfg.microvm.macvtap; link = interface;
mode = "bridge"; mode = "bridge";
}; };
} }
]; );
shares = shares =
[ [
@ -73,9 +78,13 @@ in {
); );
}; };
networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac; networking.renameInterfacesByMac = flip mapAttrs guestCfg.microvm.interfaces (_: {mac, ...}: mac);
systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce { systemd.network.networks = flip concatMapAttrs guestCfg.microvm.interfaces (
MACAddress = guestCfg.microvm.mac; name: {mac, ...}: {
}; "10-${name}".matchConfig = mkForce {
MACAddress = mac;
};
}
);
}; };
} }