mirror of
https://github.com/oddlama/nix-config.git
synced 2025-10-11 07:10:39 +02:00
refactor: move guest system and common lib parts to extra-modules
This commit is contained in:
parent
ab9c6fc507
commit
a44f73d3b0
16 changed files with 72 additions and 1092 deletions
|
@ -26,8 +26,6 @@
|
|||
./config/system.nix
|
||||
./config/users.nix
|
||||
|
||||
./guests
|
||||
|
||||
./acme-wildcard.nix
|
||||
./deterministic-ids.nix
|
||||
./distributed-config.nix
|
||||
|
@ -45,7 +43,6 @@
|
|||
];
|
||||
|
||||
nixpkgs.overlays = [
|
||||
inputs.microvm.overlay
|
||||
inputs.nixpkgs-wayland.overlay
|
||||
inputs.nixvim.overlays.default
|
||||
inputs.wired-notify.overlays.default
|
||||
|
|
|
@ -1,39 +0,0 @@
|
|||
_guestName: guestCfg: {
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) mkForce;
|
||||
in {
|
||||
node.name = guestCfg.nodeName;
|
||||
node.type = guestCfg.backend;
|
||||
|
||||
# Set early hostname too, so we can associate those logs to this host and don't get "localhost" entries in loki
|
||||
boot.kernelParams = lib.mkIf (!config.boot.isContainer) [
|
||||
"systemd.hostname=${config.networking.hostName}"
|
||||
];
|
||||
|
||||
nix = {
|
||||
settings.auto-optimise-store = mkForce false;
|
||||
optimise.automatic = mkForce false;
|
||||
gc.automatic = mkForce false;
|
||||
};
|
||||
|
||||
systemd.network.networks."10-${guestCfg.networking.mainLinkName}" = {
|
||||
matchConfig.Name = guestCfg.networking.mainLinkName;
|
||||
DHCP = "yes";
|
||||
dhcpV4Config.UseDNS = false;
|
||||
dhcpV6Config.UseDNS = false;
|
||||
ipv6AcceptRAConfig.UseDNS = false;
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
MulticastDNS = true;
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
|
||||
networking.nftables.firewall = {
|
||||
zones.untrusted.interfaces = [guestCfg.networking.mainLinkName];
|
||||
};
|
||||
}
|
|
@ -1,68 +0,0 @@
|
|||
guestName: guestCfg: {
|
||||
config,
|
||||
inputs,
|
||||
lib,
|
||||
minimal,
|
||||
nodes,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit
|
||||
(lib)
|
||||
flip
|
||||
mapAttrs'
|
||||
nameValuePair
|
||||
;
|
||||
in {
|
||||
ephemeral = true;
|
||||
privateNetwork = true;
|
||||
autoStart = guestCfg.autostart;
|
||||
macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"];
|
||||
extraFlags = [
|
||||
"--uuid=${builtins.substring 0 32 (builtins.hashString "sha256" guestName)}"
|
||||
];
|
||||
bindMounts = flip mapAttrs' guestCfg.zfs (
|
||||
_: zfsCfg:
|
||||
nameValuePair zfsCfg.guestMountpoint {
|
||||
hostPath = zfsCfg.hostMountpoint;
|
||||
isReadOnly = false;
|
||||
}
|
||||
);
|
||||
nixosConfiguration = inputs.nixpkgs.lib.nixosSystem {
|
||||
specialArgs = {
|
||||
inherit lib nodes inputs minimal;
|
||||
};
|
||||
prefix = ["nodes" "${config.node.name}-${guestName}" "config"];
|
||||
system = null;
|
||||
modules =
|
||||
[
|
||||
{
|
||||
boot.isContainer = true;
|
||||
networking.useHostResolvConf = false;
|
||||
|
||||
# We cannot force the package set via nixpkgs.pkgs and
|
||||
# inputs.nixpkgs.nixosModules.readOnlyPkgs, since some nixosModules
|
||||
# like nixseparatedebuginfod depend on adding packages via nixpkgs.overlays.
|
||||
# So we just mimic the options and overlays defined by the passed pkgs set.
|
||||
nixpkgs.hostPlatform = config.nixpkgs.hostPlatform.system;
|
||||
nixpkgs.overlays = pkgs.overlays;
|
||||
nixpkgs.config = pkgs.config;
|
||||
|
||||
# Bind the /guest/* paths from above so impermancence doesn't complain.
|
||||
# We bind-mount stuff from the host to itself, which is perfectly defined
|
||||
# and not recursive. This allows us to have a fileSystems entry for each
|
||||
# bindMount which other stuff can depend upon (impermanence adds dependencies
|
||||
# to the state fs).
|
||||
fileSystems = flip mapAttrs' guestCfg.zfs (_: zfsCfg:
|
||||
nameValuePair zfsCfg.guestMountpoint {
|
||||
neededForBoot = true;
|
||||
fsType = "none";
|
||||
device = zfsCfg.guestMountpoint;
|
||||
options = ["bind"];
|
||||
});
|
||||
}
|
||||
(import ./common-guest-config.nix guestName guestCfg)
|
||||
]
|
||||
++ guestCfg.modules;
|
||||
};
|
||||
}
|
|
@ -1,261 +0,0 @@
|
|||
{
|
||||
config,
|
||||
inputs,
|
||||
lib,
|
||||
pkgs,
|
||||
utils,
|
||||
...
|
||||
} @ attrs: let
|
||||
inherit
|
||||
(lib)
|
||||
attrNames
|
||||
attrValues
|
||||
attrsToList
|
||||
disko
|
||||
escapeShellArg
|
||||
flip
|
||||
groupBy
|
||||
listToAttrs
|
||||
makeBinPath
|
||||
mapAttrs
|
||||
mapAttrsToList
|
||||
mergeToplevelConfigs
|
||||
mkIf
|
||||
mkMerge
|
||||
mkOption
|
||||
net
|
||||
types
|
||||
;
|
||||
|
||||
backends = ["microvm" "container"];
|
||||
nodeName = config.node.name;
|
||||
guestsByBackend =
|
||||
lib.genAttrs backends (_: {})
|
||||
// mapAttrs (_: listToAttrs) (groupBy (x: x.value.backend) (attrsToList config.guests));
|
||||
|
||||
# List the necessary mount units for the given guest
|
||||
fsMountUnitsFor = guestCfg:
|
||||
map
|
||||
(x: "${utils.escapeSystemdPath x.hostMountpoint}.mount")
|
||||
(attrValues guestCfg.zfs);
|
||||
|
||||
# Configuration required on the host for a specific guest
|
||||
defineGuest = _guestName: guestCfg: {
|
||||
# Add the required datasets to the disko configuration of the machine
|
||||
disko.devices.zpool = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: {
|
||||
${zfsCfg.pool}.datasets.${zfsCfg.dataset} =
|
||||
disko.zfs.filesystem zfsCfg.hostMountpoint;
|
||||
}));
|
||||
|
||||
# Ensure that the zfs dataset exists before it is mounted.
|
||||
systemd.services = mkMerge (flip map (attrValues guestCfg.zfs) (zfsCfg: let
|
||||
fsMountUnit = "${utils.escapeSystemdPath zfsCfg.hostMountpoint}.mount";
|
||||
in {
|
||||
"zfs-ensure-${utils.escapeSystemdPath zfsCfg.hostMountpoint}" = {
|
||||
wantedBy = [fsMountUnit];
|
||||
before = [fsMountUnit];
|
||||
after = [
|
||||
"zfs-import-${utils.escapeSystemdPath zfsCfg.pool}.service"
|
||||
"zfs-mount.target"
|
||||
];
|
||||
unitConfig.DefaultDependencies = "no";
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = let
|
||||
poolDataset = "${zfsCfg.pool}/${zfsCfg.dataset}";
|
||||
diskoDataset = config.disko.devices.zpool.${zfsCfg.pool}.datasets.${zfsCfg.dataset};
|
||||
in ''
|
||||
export PATH=${makeBinPath [pkgs.zfs]}":$PATH"
|
||||
if ! zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
|
||||
${diskoDataset._create}
|
||||
fi
|
||||
'';
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
||||
defineMicrovm = guestName: guestCfg: {
|
||||
# Ensure that the zfs dataset exists before it is mounted.
|
||||
systemd.services."microvm@${guestName}" = {
|
||||
requires = fsMountUnitsFor guestCfg;
|
||||
after = fsMountUnitsFor guestCfg;
|
||||
};
|
||||
|
||||
microvm.vms.${guestName} = import ./microvm.nix guestName guestCfg attrs;
|
||||
};
|
||||
|
||||
defineContainer = guestName: guestCfg: {
|
||||
# Ensure that the zfs dataset exists before it is mounted.
|
||||
systemd.services."container@${guestName}" = {
|
||||
requires = fsMountUnitsFor guestCfg;
|
||||
after = fsMountUnitsFor guestCfg;
|
||||
# Don't use the notify service type. Using exec will always consider containers
|
||||
# started immediately and donesn't wait until the container is fully booted.
|
||||
# Containers should behave like independent machines, and issues inside the container
|
||||
# will unnecessarily lock up the service on the host otherwise.
|
||||
# This causes issues on system activation or when containers take longer to start
|
||||
# than TimeoutStartSec.
|
||||
serviceConfig.Type = lib.mkForce "exec";
|
||||
};
|
||||
|
||||
containers.${guestName} = import ./container.nix guestName guestCfg attrs;
|
||||
};
|
||||
in {
|
||||
imports = [
|
||||
# Add the host module, but only enable if it necessary
|
||||
inputs.microvm.nixosModules.host
|
||||
# This is opt-out, so we can't put this into the mkIf below
|
||||
{
|
||||
microvm.host.enable = guestsByBackend.microvm != {};
|
||||
}
|
||||
];
|
||||
|
||||
options.node.type = mkOption {
|
||||
type = types.enum ["host" "microvm" "container"];
|
||||
description = "The type of this machine.";
|
||||
default = "host";
|
||||
};
|
||||
|
||||
options.containers = mkOption {
|
||||
type = types.attrsOf (types.submodule (submod: {
|
||||
options.nixosConfiguration = mkOption {
|
||||
type = types.unspecified;
|
||||
default = null;
|
||||
description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you.";
|
||||
};
|
||||
config = mkIf (submod.config.nixosConfiguration != null) {
|
||||
path = submod.config.nixosConfiguration.config.system.build.toplevel;
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
||||
options.guests = mkOption {
|
||||
default = {};
|
||||
description = "Defines the actual vms and handles the necessary base setup for them.";
|
||||
type = types.attrsOf (types.submodule (submod: {
|
||||
options = {
|
||||
nodeName = mkOption {
|
||||
type = types.str;
|
||||
default = "${nodeName}-${submod.config._module.args.name}";
|
||||
description = ''
|
||||
The name of the resulting node. By default this will be a compound name
|
||||
of the host's name and the vm's name to avoid name clashes. Can be
|
||||
overwritten to designate special names to specific vms.
|
||||
'';
|
||||
};
|
||||
|
||||
backend = mkOption {
|
||||
type = types.enum backends;
|
||||
description = ''
|
||||
Determines how the guest will be hosted. You can currently choose
|
||||
between microvm based deployment, or nixos containers.
|
||||
'';
|
||||
};
|
||||
|
||||
# Options for the microvm backend
|
||||
microvm = {
|
||||
system = mkOption {
|
||||
type = types.str;
|
||||
description = "The system that this microvm should use";
|
||||
};
|
||||
|
||||
macvtap = mkOption {
|
||||
type = types.str;
|
||||
description = "The host interface to which the microvm should be attached via macvtap";
|
||||
};
|
||||
|
||||
baseMac = mkOption {
|
||||
type = types.net.mac;
|
||||
description = "The base mac address from which the guest's mac will be derived. Only the second and third byte are used, so for 02:XX:YY:ZZ:ZZ:ZZ, this specifies XX and YY, while Zs are generated automatically. Not used if the mac is set directly.";
|
||||
default = "02:01:27:00:00:00";
|
||||
};
|
||||
|
||||
mac = mkOption {
|
||||
type = types.net.mac;
|
||||
description = "The MAC address for the guest's macvtap interface";
|
||||
default = let
|
||||
base = "02:${lib.substring 3 5 submod.config.microvm.baseMac}:00:00:00";
|
||||
in
|
||||
(net.mac.assignMacs base 24 [] (attrNames config.guests)).${submod.config._module.args.name};
|
||||
};
|
||||
};
|
||||
|
||||
# Options for the container backend
|
||||
container = {
|
||||
macvlan = mkOption {
|
||||
type = types.str;
|
||||
description = "The host interface to which the container should be attached";
|
||||
};
|
||||
};
|
||||
|
||||
networking.mainLinkName = mkOption {
|
||||
type = types.str;
|
||||
description = "The main ethernet link name inside of the guest. For containers, this cannot be named similar to an existing interface on the host.";
|
||||
default =
|
||||
if submod.config.backend == "microvm"
|
||||
then submod.config.microvm.macvtap
|
||||
else if submod.config.backend == "container"
|
||||
then "mv-${submod.config.container.macvlan}"
|
||||
else throw "Invalid backend";
|
||||
};
|
||||
|
||||
zfs = mkOption {
|
||||
description = "zfs datasets to mount into the guest";
|
||||
default = {};
|
||||
type = types.attrsOf (types.submodule (zfsSubmod: {
|
||||
options = {
|
||||
pool = mkOption {
|
||||
type = types.str;
|
||||
description = "The host's zfs pool on which the dataset resides";
|
||||
};
|
||||
|
||||
dataset = mkOption {
|
||||
type = types.str;
|
||||
example = "safe/guests/mycontainer";
|
||||
description = "The host's dataset that should be used for this mountpoint (will automatically be created, including parent datasets)";
|
||||
};
|
||||
|
||||
hostMountpoint = mkOption {
|
||||
type = types.path;
|
||||
default = "/guests/${submod.config._module.args.name}${zfsSubmod.config._module.args.name}";
|
||||
example = "/guests/mycontainer/persist";
|
||||
description = "The host's mountpoint for the guest's dataset";
|
||||
};
|
||||
|
||||
guestMountpoint = mkOption {
|
||||
type = types.path;
|
||||
default = zfsSubmod.config._module.args.name;
|
||||
example = "/persist";
|
||||
description = "The mountpoint inside the guest.";
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
||||
autostart = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Whether this VM should be started automatically with the host";
|
||||
};
|
||||
|
||||
modules = mkOption {
|
||||
type = types.listOf types.unspecified;
|
||||
default = [];
|
||||
description = "Additional modules to load";
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
||||
config = mkIf (config.guests != {}) (
|
||||
mkMerge [
|
||||
{
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /guests 0700 root root -"
|
||||
];
|
||||
}
|
||||
(mergeToplevelConfigs ["disko" "systemd"] (mapAttrsToList defineGuest config.guests))
|
||||
(mergeToplevelConfigs ["containers" "systemd"] (mapAttrsToList defineContainer guestsByBackend.container))
|
||||
(mergeToplevelConfigs ["microvm" "systemd"] (mapAttrsToList defineMicrovm guestsByBackend.microvm))
|
||||
]
|
||||
);
|
||||
}
|
|
@ -1,79 +0,0 @@
|
|||
guestName: guestCfg: {
|
||||
config,
|
||||
inputs,
|
||||
lib,
|
||||
pkgs,
|
||||
minimal,
|
||||
...
|
||||
}: let
|
||||
inherit
|
||||
(lib)
|
||||
flip
|
||||
mapAttrsToList
|
||||
mkDefault
|
||||
mkForce
|
||||
;
|
||||
in {
|
||||
specialArgs = {
|
||||
inherit (inputs.self) nodes;
|
||||
inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib;
|
||||
inherit inputs minimal;
|
||||
};
|
||||
pkgs = inputs.self.pkgs.${guestCfg.microvm.system};
|
||||
inherit (guestCfg) autostart;
|
||||
config = {
|
||||
imports = guestCfg.modules ++ [(import ./common-guest-config.nix guestName guestCfg)];
|
||||
|
||||
# TODO needed because of https://github.com/NixOS/nixpkgs/issues/102137
|
||||
environment.noXlibs = mkForce false;
|
||||
lib.microvm.mac = guestCfg.microvm.mac;
|
||||
|
||||
microvm = {
|
||||
hypervisor = mkDefault "qemu";
|
||||
|
||||
# Give them some juice by default
|
||||
mem = mkDefault (1024 + 2048);
|
||||
|
||||
# Add a writable store overlay, but since this is always ephemeral
|
||||
# disable any store optimization from nix.
|
||||
writableStoreOverlay = "/nix/.rw-store";
|
||||
|
||||
# MACVTAP bridge to the host's network
|
||||
interfaces = [
|
||||
{
|
||||
type = "macvtap";
|
||||
id = "vm-${guestName}";
|
||||
inherit (guestCfg.microvm) mac;
|
||||
macvtap = {
|
||||
link = guestCfg.microvm.macvtap;
|
||||
mode = "bridge";
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
shares =
|
||||
[
|
||||
# Share the nix-store of the host
|
||||
{
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "ro-store";
|
||||
proto = "virtiofs";
|
||||
}
|
||||
]
|
||||
++ flip mapAttrsToList guestCfg.zfs (
|
||||
_: zfsCfg: {
|
||||
source = zfsCfg.hostMountpoint;
|
||||
mountPoint = zfsCfg.guestMountpoint;
|
||||
tag = builtins.substring 0 16 (builtins.hashString "sha256" zfsCfg.hostMountpoint);
|
||||
proto = "virtiofs";
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
networking.renameInterfacesByMac.${guestCfg.networking.mainLinkName} = guestCfg.microvm.mac;
|
||||
systemd.network.networks."10-${guestCfg.networking.mainLinkName}".matchConfig = mkForce {
|
||||
MACAddress = guestCfg.microvm.mac;
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,27 +1,12 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
{lib, ...}: let
|
||||
inherit
|
||||
(lib)
|
||||
mkOption
|
||||
types
|
||||
;
|
||||
in {
|
||||
options.node = {
|
||||
name = mkOption {
|
||||
description = "A unique name for this node (host) in the repository. Defines the default hostname, but this can be overwritten.";
|
||||
type = types.str;
|
||||
};
|
||||
|
||||
secretsDir = mkOption {
|
||||
description = "Path to the secrets directory for this node.";
|
||||
type = types.path;
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
networking.hostName = config.node.name;
|
||||
options.node.secretsDir = mkOption {
|
||||
description = "Path to the secrets directory for this node.";
|
||||
type = types.path;
|
||||
};
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue