diff --git a/modules/guests/common-guest-config.nix b/modules/guests/common-guest-config.nix index 03df8d6..a88e6bf 100644 --- a/modules/guests/common-guest-config.nix +++ b/modules/guests/common-guest-config.nix @@ -1,5 +1,11 @@ _guestName: guestCfg: {lib, ...}: let - inherit (lib) mkForce; + inherit + (lib) + mkForce + nameValuePair + listToAttrs + flip + ; in { node.name = guestCfg.nodeName; node.type = guestCfg.backend; @@ -11,18 +17,23 @@ in { }; documentation.enable = mkForce false; - systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = { - matchConfig.Name = guestCfg.networking.mainLinkName; - DHCP = "yes"; - # XXX: Do we really want this? - dhcpV4Config.UseDNS = false; - dhcpV6Config.UseDNS = false; - ipv6AcceptRAConfig.UseDNS = false; - networkConfig = { - IPv6PrivacyExtensions = "yes"; - MulticastDNS = true; - IPv6AcceptRA = true; - }; - linkConfig.RequiredForOnline = "routable"; - }; + systemd.network.networks = listToAttrs ( + flip map guestCfg.networking.links ( + name: + nameValuePair "10-${name}" { + matchConfig.Name = name; + DHCP = "yes"; + # XXX: Do we really want this? + dhcpV4Config.UseDNS = false; + dhcpV6Config.UseDNS = false; + ipv6AcceptRAConfig.UseDNS = false; + networkConfig = { + IPv6PrivacyExtensions = "yes"; + MulticastDNS = true; + IPv6AcceptRA = true; + }; + linkConfig.RequiredForOnline = "routable"; + } + ) + ); } diff --git a/modules/guests/container.nix b/modules/guests/container.nix index 85d61a9..cee541b 100644 --- a/modules/guests/container.nix +++ b/modules/guests/container.nix @@ -12,10 +12,10 @@ guestName: guestCfg: { nameValuePair ; in { + inherit (guestCfg.container) macvlans; ephemeral = true; privateNetwork = true; autoStart = guestCfg.autostart; - macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"]; extraFlags = [ "--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}" ]; @@ -28,7 +28,11 @@ in { ); nixosConfiguration = (import "${inputs.nixpkgs}/nixos/lib/eval-config.nix") { specialArgs = guestCfg.extraSpecialArgs; - prefix = ["nodes" "${config.node.name}-${guestName}" "config"]; + prefix = [ + "nodes" + "${config.node.name}-${guestName}" + "config" + ]; system = null; modules = [ @@ -49,13 +53,15 @@ in { # and not recursive. This allows us to have a fileSystems entry for each # bindMount which other stuff can depend upon (impermanence adds dependencies # to the state fs). - fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg: - nameValuePair zfsCfg.guestMountpoint { - neededForBoot = true; - fsType = "none"; - device = zfsCfg.guestMountpoint; - options = ["bind"]; - }); + fileSystems = flip mapAttrs' guestCfg.zfs ( + _: zfsCfg: + nameValuePair zfsCfg.guestMountpoint { + neededForBoot = true; + fsType = "none"; + device = zfsCfg.guestMountpoint; + options = ["bind"]; + } + ); } (import ./common-guest-config.nix guestName guestCfg) ] diff --git a/modules/guests/default.nix b/modules/guests/default.nix index 44baf0f..80db630 100644 --- a/modules/guests/default.nix +++ b/modules/guests/default.nix @@ -10,11 +10,15 @@ attrNames attrValues attrsToList + length + splitString + elemAt disko escapeShellArg flatten flip foldl' + forEach groupBy hasInfix hasPrefix @@ -35,51 +39,57 @@ ; # All available backends - backends = ["microvm" "container"]; + backends = [ + "microvm" + "container" + ]; guestsByBackend = lib.genAttrs backends (_: {}) // mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests)); # List the necessary mount units for the given guest - fsMountUnitsFor = guestCfg: - map - (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") - (attrValues guestCfg.zfs); + fsMountUnitsFor = guestCfg: map (x: "${utils.escapeSystemdPath x.hostMountpoint}.mount") (attrValues guestCfg.zfs); # Configuration required on the host for a specific guest defineGuest = _guestName: guestCfg: { # Add the required datasets to the disko configuration of the machine - disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: { - ${zfsCfg.pool}.datasets.${zfsCfg.dataset} = - # We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests - disko.zfs.unmountable; - })); + disko.devices.zpool = mkMerge ( + flip map (attrValues guestCfg.zfs) (zfsCfg: { + ${zfsCfg.pool}.datasets.${zfsCfg.dataset} = + # We generate the mountpoint fileSystems entries ourselfs to enable shared folders between guests + disko.zfs.unmountable; + }) + ); # Ensure that the zfs dataset exists before it is mounted. - systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let - fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount"; - in { - "zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = { - wantedBy = [fsMountUnit]; - before = [fsMountUnit]; - after = [ - "zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service" - "zfs-mount.target" - ]; - unitConfig.DefaultDependencies = "no"; - serviceConfig.Type = "oneshot"; - script = let - poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}"; - diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset}; - in '' - export PATH=${makeBinPath [pkgs.zfs]}":$PATH" - if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then - ${diskoDataset._create} - fi - ''; - }; - })); + systemd.services = mkMerge ( + flip map (attrValues guestCfg.zfs) ( + zfsCfg: let + fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount"; + in { + "zfs-ensure-${utils.escapeSystemdPath "${zfsCfg.pool}/${zfsCfg.dataset}"}" = { + wantedBy = [fsMountUnit]; + before = [fsMountUnit]; + after = [ + "zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service" + "zfs-mount.target" + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig.Type = "oneshot"; + script = let + poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}"; + diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset}; + in '' + export PATH=${makeBinPath [pkgs.zfs]}":$PATH" + if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then + ${diskoDataset._create} + fi + ''; + }; + } + ) + ); }; defineMicrovm = guestName: guestCfg: { @@ -123,159 +133,188 @@ in { }; options.containers = mkOption { - type = types.attrsOf (types.submodule (submod: { - options.nixosConfiguration = mkOption { - type = types.unspecified; - default = null; - description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you."; - }; - config = mkIf (submod.config.nixosConfiguration != null) ({ - path = submod.config.nixosConfiguration.config.system.build.toplevel; - } - // optionalAttrs (config ? topology) { - _nix_topology_config = submod.config.nixosConfiguration.config; - }); - })); + type = types.attrsOf ( + types.submodule (submod: { + options.nixosConfiguration = mkOption { + type = types.unspecified; + default = null; + description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you."; + }; + config = mkIf (submod.config.nixosConfiguration != null) ( + { + path = submod.config.nixosConfiguration.config.system.build.toplevel; + } + // optionalAttrs (config ? topology) { + _nix_topology_config = submod.config.nixosConfiguration.config; + } + ); + }) + ); }; options.guests = mkOption { default = {}; description = "Defines the actual vms and handles the necessary base setup for them."; - type = types.attrsOf (types.submodule (submod: { - options = { - nodeName = mkOption { - type = types.str; - default = "${config.node.name}-${submod.config._module.args.name}"; - description = '' - The name of the resulting node. By default this will be a compound name - of the host's name and the guest's name to avoid name clashes. Can be - overwritten to designate special names to specific guests. - ''; - }; - - backend = mkOption { - type = types.enum backends; - description = '' - Determines how the guest will be hosted. You can currently choose - between microvm based deployment, or nixos containers. - ''; - }; - - extraSpecialArgs = mkOption { - type = types.attrs; - default = {}; - example = literalExpression "{ inherit inputs; }"; - description = '' - Extra `specialArgs` passed to each guest system definition. This - option can be used to pass additional arguments to all modules. - ''; - }; - - # Options for the microvm backend - microvm = { - system = mkOption { + type = types.attrsOf ( + types.submodule (submod: { + options = { + nodeName = mkOption { type = types.str; - description = "The system that this microvm should use"; + default = "${config.node.name}-${submod.config._module.args.name}"; + description = '' + The name of the resulting node. By default this will be a compound name + of the host's name and the guest's name to avoid name clashes. Can be + overwritten to designate special names to specific guests. + ''; }; - macvtap = mkOption { - type = types.str; - description = "The host interface to which the microvm should be attached via macvtap"; + backend = mkOption { + type = types.enum backends; + description = '' + Determines how the guest will be hosted. You can currently choose + between microvm based deployment, or nixos containers. + ''; }; - baseMac = mkOption { - type = types.net.mac; - description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly."; - default = "02:01:27:00:00:00"; + extraSpecialArgs = mkOption { + type = types.attrs; + default = {}; + example = literalExpression "{ inherit inputs; }"; + description = '' + Extra `specialArgs` passed to each guest system definition. This + option can be used to pass additional arguments to all modules. + ''; }; - mac = mkOption { - type = types.net.mac; - description = "The MAC address for the guest's macvtap interface"; - default = let - base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00"; - in - (net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name}; - }; - }; - - # Options for the container backend - container = { - macvlan = mkOption { - type = types.str; - description = "The host interface to which the container should be attached"; - }; - }; - - networking.mainLinkName = mkOption { - type = types.str; - description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host."; - default = - if submod.config.backend == "microvm" - then submod.config.microvm.macvtap - else if submod.config.backend == "container" - then "mv-${submod.config.container.macvlan}" - else throw "Invalid backend"; - }; - - zfs = mkOption { - description = "zfs datasets to mount into the guest"; - default = {}; - type = types.attrsOf (types.submodule (zfsSubmod: { - options = { - pool = mkOption { - type = types.str; - description = "The host's zfs pool on which the dataset resides"; - }; - - dataset = mkOption { - type = types.str; - example = "safe/guests/mycontainer"; - description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)"; - }; - - hostMountpoint = mkOption { - type = types.path; - default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}"; - example = "/guests/mycontainer/persist"; - description = "The host's mountpoint for the guest's dataset"; - }; - - guestMountpoint = mkOption { - type = types.path; - default = zfsSubmod.config._module.args.name; - example = "/persist"; - description = "The mountpoint inside the guest."; - }; + # Options for the microvm backend + microvm = { + system = mkOption { + type = types.str; + description = "The system that this microvm should use"; }; - })); - }; - autostart = mkOption { - type = types.bool; - default = false; - description = "Whether this guest should be started automatically with the host"; - }; + baseMac = mkOption { + type = types.net.mac; + description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly."; + default = "02:01:27:00:00:00"; + }; + interfaces = mkOption { + description = "An attrset correlating the host interface to which the microvm should be attached via macvtap, with its mac address"; + type = types.attrsOf ( + types.submodule (submod-iface: { + options.mac = mkOption { + type = types.net.mac; + description = "The MAC address for the guest's macvtap interface"; + default = let + base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00"; + in + (net.mac.assignMacs base 24 [] ( + flatten ( + flip mapAttrsToList config.guests ( + name: value: forEach (attrNames value.microvm.interfaces) (iface: "${name}-${iface}") + ) + ) + )) + ."${submod.config._module.args.name}-${submod-iface.config._module.args.name}"; + }; + }) + ); + default = {}; + }; + }; - modules = mkOption { - type = types.listOf types.unspecified; - default = []; - description = "Additional modules to load"; + # Options for the container backend + container = { + macvlans = mkOption { + type = types.listOf types.str; + description = '' + The macvlans to be created for the container. + Can be either an interface name in which case the container interface will be called mv- or a pair + of :. + ''; + }; + }; + + networking.links = mkOption { + type = types.listOf types.str; + description = "The ethernet links inside of the guest. For containers, these cannot be named similar to an existing interface on the host."; + default = + if submod.config.backend == "microvm" + then (flip mapAttrsToList submod.config.microvm.interfaces (name: _: name)) + else if submod.config.backend == "container" + then + (forEach submod.config.container.macvlans ( + name: let + split = splitString ":" name; + in + if length split > 1 + then elemAt split 1 + else "mv-${name}" + )) + else throw "Invalid backend"; + }; + + zfs = mkOption { + description = "zfs datasets to mount into the guest"; + default = {}; + type = types.attrsOf ( + types.submodule (zfsSubmod: { + options = { + pool = mkOption { + type = types.str; + description = "The host's zfs pool on which the dataset resides"; + }; + + dataset = mkOption { + type = types.str; + example = "safe/guests/mycontainer"; + description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)"; + }; + + hostMountpoint = mkOption { + type = types.path; + default = "/guests/${submod.config._module.args.name}${zfsSubmod.config.guestMountpoint}"; + example = "/guests/mycontainer/persist"; + description = "The host's mountpoint for the guest's dataset"; + }; + + guestMountpoint = mkOption { + type = types.path; + default = zfsSubmod.config._module.args.name; + example = "/persist"; + description = "The mountpoint inside the guest."; + }; + }; + }) + ); + }; + + autostart = mkOption { + type = types.bool; + default = false; + description = "Whether this guest should be started automatically with the host"; + }; + + modules = mkOption { + type = types.listOf types.unspecified; + default = []; + description = "Additional modules to load"; + }; }; - }; - })); + }) + ); }; - config = mkIf (config.guests != {}) ( - mkMerge [ - { - systemd.tmpfiles.rules = [ - "d /guests 0700 root root -" - ]; + config = mkIf (config.guests != {}) (mkMerge [ + { + systemd.tmpfiles.rules = [ + "d /guests 0700 root root -" + ]; - # To enable shared folders we need to do all fileSystems entries ourselfs - fileSystems = let - zfsDefs = flatten (flip mapAttrsToList config.guests ( + # To enable shared folders we need to do all fileSystems entries ourselfs + fileSystems = let + zfsDefs = flatten ( + flip mapAttrsToList config.guests ( _: guestCfg: flip mapAttrsToList guestCfg.zfs ( _: zfsCfg: { @@ -283,39 +322,53 @@ in { inherit (zfsCfg) hostMountpoint; } ) - )); - # Due to limitations in zfs mounting we need to explicitly set an order in which - # any dataset gets mounted - zfsDefsByPath = flip groupBy zfsDefs (x: x.path); - in - mkMerge (flip mapAttrsToList zfsDefsByPath (_: defs: - (foldl' ({ - prev, - res, - }: elem: { - prev = elem; - res = - res - // { - ${elem.hostMountpoint} = { - fsType = "zfs"; - options = - ["zfsutil"] - ++ optional (prev != null) "x-systemd.requires-mounts-for=${warnIf - (hasInfix " " prev.hostMountpoint) "HostMountpoint ${prev.hostMountpoint} cannot contain a space" - prev.hostMountpoint}"; - device = elem.path; - }; - }; - }) - { - prev = null; - res = {}; - } - defs) - .res)); + ) + ); + # Due to limitations in zfs mounting we need to explicitly set an order in which + # any dataset gets mounted + zfsDefsByPath = flip groupBy zfsDefs (x: x.path); + in + mkMerge ( + flip mapAttrsToList zfsDefsByPath ( + _: defs: + ( + foldl' + ( + { + prev, + res, + }: elem: { + prev = elem; + res = + res + // { + ${elem.hostMountpoint} = { + fsType = "zfs"; + options = + ["zfsutil"] + ++ optional (prev != null) + "x-systemd.requires-mounts-for=${ + warnIf (hasInfix " " prev.hostMountpoint) + "HostMountpoint ${prev.hostMountpoint} cannot contain a space" + prev.hostMountpoint + }"; + device = elem.path; + }; + }; + } + ) + { + prev = null; + res = {}; + } + defs + ) + .res + ) + ); - assertions = flatten (flip mapAttrsToList config.guests ( + assertions = flatten ( + flip mapAttrsToList config.guests ( guestName: guestCfg: flip mapAttrsToList guestCfg.zfs ( zfsName: zfsCfg: { @@ -323,11 +376,17 @@ in { message = "guest ${guestName}: zfs ${zfsName}: the guestMountpoint must be an absolute path."; } ) - )); - } - (mergeToplevelConfigs ["disko" "systemd" "fileSystems"] (mapAttrsToList defineGuest config.guests)) - (mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container)) - (mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm)) - ] - ); + ) + ); + } + (mergeToplevelConfigs ["disko" "systemd" "fileSystems"] ( + mapAttrsToList defineGuest config.guests + )) + (mergeToplevelConfigs ["containers" "systemd"] ( + mapAttrsToList defineContainer guestsByBackend.container + )) + (mergeToplevelConfigs ["microvm" "systemd"] ( + mapAttrsToList defineMicrovm guestsByBackend.microvm + )) + ]); } diff --git a/modules/guests/microvm.nix b/modules/guests/microvm.nix index 6d65f25..d67432b 100644 --- a/modules/guests/microvm.nix +++ b/modules/guests/microvm.nix @@ -5,10 +5,13 @@ guestName: guestCfg: { }: let inherit (lib) + concatMapAttrs flip + mapAttrs mapAttrsToList mkDefault mkForce + replaceStrings ; in { specialArgs = guestCfg.extraSpecialArgs; @@ -19,13 +22,15 @@ in { guestCfg.modules ++ [ (import ./common-guest-config.nix guestName guestCfg) - ({config, ...}: { - # Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki - boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"]; - }) + ( + {config, ...}: { + # Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki + boot.kernelParams = ["systemd.hostname=${config.networking.hostName}"]; + } + ) ]; - lib.microvm.mac = guestCfg.microvm.mac; + lib.microvm.interfaces = guestCfg.microvm.interfaces; microvm = { hypervisor = mkDefault "qemu"; @@ -41,17 +46,17 @@ in { writableStoreOverlay = "/nix/.rw-store"; # MACVTAP bridge to the host's network - interfaces = [ - { + interfaces = flip mapAttrsToList guestCfg.microvm.interfaces ( + interface: {mac, ...}: { type = "macvtap"; - id = "vm-${guestName}"; - inherit (guestCfg.microvm) mac; + id = "vm-${replaceStrings [":"] [""] mac}"; + inherit mac; macvtap = { - link = guestCfg.microvm.macvtap; + link = interface; mode = "bridge"; }; } - ]; + ); shares = [ @@ -73,9 +78,13 @@ in { ); }; - networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac; - systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce { - MACAddress = guestCfg.microvm.mac; - }; + networking.renameInterfacesByMac = flip mapAttrs guestCfg.microvm.interfaces (_: {mac, ...}: mac); + systemd.network.networks = flip concatMapAttrs guestCfg.microvm.interfaces ( + name: {mac, ...}: { + "10-${name}".matchConfig = mkForce { + MACAddress = mac; + }; + } + ); }; }