mirror of
https://github.com/oddlama/nix-config.git
synced 2025-10-11 07:10:39 +02:00
feat: add automatic zfs persistent dataset management to microvms
This commit is contained in:
oddlama
parent
2b4449569f
commit
70f7ef3023
5 changed files with 166 additions and 91 deletions
|
|
@ -21,8 +21,9 @@
|
|||
"/etc/ssh/ssh_host_ed25519_key.pub"
|
||||
];
|
||||
directories = [
|
||||
"/var/log"
|
||||
"/var/lib/nixos"
|
||||
"/var/lib/systemd/coredump"
|
||||
"/var/log"
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,12 @@
|
|||
macOffset = config.lib.net.mac.addPrivate nodeSecrets.networking.interfaces.lan.mac;
|
||||
in {
|
||||
test = {
|
||||
zfs = {
|
||||
enable = true;
|
||||
pool = "rpool";
|
||||
dataset = "safe/vms/test";
|
||||
mountpoint = "/persist/vms/test";
|
||||
};
|
||||
autostart = true;
|
||||
mac = macOffset "00:00:00:00:00:11";
|
||||
macvtap = "lan";
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@
|
|||
"local/nix" = filesystem "/nix";
|
||||
"safe" = unmountable;
|
||||
"safe/persist" = filesystem "/persist";
|
||||
"safe/vms" = unmountable;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
@ -42,10 +43,6 @@
|
|||
|
||||
fileSystems."/persist".neededForBoot = true;
|
||||
|
||||
#environment.persistence."/persist".directories = [
|
||||
# { directory = "/var/lib/acme"; user = "acme"; group = "acme"; }
|
||||
#];
|
||||
|
||||
# After importing the rpool, rollback the root system to be empty.
|
||||
boot.initrd.systemd.services = {
|
||||
impermanence-root = {
|
||||
|
|
|
|||
|
|
@ -13,100 +13,174 @@
|
|||
(lib)
|
||||
attrNames
|
||||
concatStringsSep
|
||||
escapeShellArg
|
||||
filterAttrs
|
||||
mapAttrs
|
||||
foldl'
|
||||
mapAttrsToList
|
||||
mdDoc
|
||||
mkDefault
|
||||
mkEnableOption
|
||||
mkForce
|
||||
mkIf
|
||||
mkMerge
|
||||
mkOption
|
||||
optionalAttrs
|
||||
recursiveUpdate
|
||||
types
|
||||
;
|
||||
|
||||
cfg = config.extra.microvms;
|
||||
|
||||
defineMicrovm = vmName: vmCfg: let
|
||||
node =
|
||||
(import ../nix/generate-node.nix inputs)
|
||||
"${nodeName}-microvm-${vmName}" {
|
||||
inherit (vmCfg) system;
|
||||
config = nodePath + "/microvms/${vmName}";
|
||||
};
|
||||
in {
|
||||
inherit (node) pkgs specialArgs;
|
||||
config = {
|
||||
imports = [microvm.microvm] ++ node.imports;
|
||||
# Base configuration required for the host
|
||||
hostConfig = {
|
||||
assertions = let
|
||||
duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
|
||||
in [
|
||||
{
|
||||
assertion = duplicateMacs == [];
|
||||
message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
|
||||
}
|
||||
];
|
||||
|
||||
microvm = {
|
||||
hypervisor = mkDefault "cloud-hypervisor";
|
||||
microvm = {
|
||||
declarativeUpdates = true;
|
||||
restartIfChanged = true;
|
||||
};
|
||||
};
|
||||
|
||||
# MACVTAP bridge to the host's network
|
||||
interfaces = [
|
||||
{
|
||||
type = "macvtap";
|
||||
id = "vm-${vmName}";
|
||||
macvtap = {
|
||||
link = vmCfg.macvtap;
|
||||
mode = "bridge";
|
||||
};
|
||||
inherit (vmCfg) mac;
|
||||
}
|
||||
];
|
||||
# Configuration for each microvm
|
||||
microvmConfig = vmName: vmCfg: {
|
||||
# Add the required datasets to the disko configuration of the machine
|
||||
disko.devices.zpool = mkIf (vmCfg.zfs.enable && vmCfg.zfs.disko) {
|
||||
${vmCfg.zfs.pool}.datasets."${vmCfg.zfs.dataset}" =
|
||||
extraLib.disko.zfs.filesystem "${vmCfg.zfs.mountpoint}";
|
||||
};
|
||||
|
||||
shares = [
|
||||
# Share the nix-store of the host
|
||||
{
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "ro-store";
|
||||
proto = "virtiofs";
|
||||
}
|
||||
# Mount persistent data from the host
|
||||
#{
|
||||
# source = "/persist/vms/${vmName}";
|
||||
# mountPoint = "/persist";
|
||||
# tag = "persist";
|
||||
# proto = "virtiofs";
|
||||
#}
|
||||
];
|
||||
};
|
||||
# When installing a microvm, make sure that its persitent zfs dataset exists
|
||||
systemd.services."install-microvm-${vmName}".preStart = let
|
||||
poolDataset = "${vmCfg.zfs.pool}/${vmCfg.zfs.dataset}";
|
||||
in
|
||||
mkIf vmCfg.zfs.enable ''
|
||||
if ! ${pkgs.zfs}/bin/zfs list -H -o type ${escapeShellArg poolDataset} &>/dev/null ; then
|
||||
${pkgs.zfs}/bin/zfs create -o canmount=on -o mountpoint=${escapeShellArg vmCfg.zfs.mountpoint} ${escapeShellArg poolDataset}
|
||||
fi
|
||||
'';
|
||||
|
||||
# Add a writable store overlay, but since this is always ephemeral
|
||||
# disable any store optimization from nix.
|
||||
microvm.writableStoreOverlay = "/nix/.rw-store";
|
||||
nix = {
|
||||
settings.auto-optimise-store = mkForce false;
|
||||
optimise.automatic = mkForce false;
|
||||
gc.automatic = mkForce false;
|
||||
};
|
||||
|
||||
extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac;
|
||||
|
||||
systemd.network.networks = {
|
||||
"10-${vmCfg.linkName}" = {
|
||||
matchConfig.Name = vmCfg.linkName;
|
||||
DHCP = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
microvm.autostart = mkIf vmCfg.autostart [vmName];
|
||||
microvm.vms.${vmName} = let
|
||||
node =
|
||||
(import ../nix/generate-node.nix inputs)
|
||||
"${nodeName}-microvm-${vmName}" {
|
||||
inherit (vmCfg) system;
|
||||
config = nodePath + "/microvms/${vmName}";
|
||||
};
|
||||
};
|
||||
in {
|
||||
inherit (node) pkgs specialArgs;
|
||||
config = {
|
||||
imports = [microvm.microvm] ++ node.imports;
|
||||
|
||||
# TODO change once microvms are compatible with stage-1 systemd
|
||||
boot.initrd.systemd.enable = mkForce false;
|
||||
microvm = {
|
||||
hypervisor = mkDefault "cloud-hypervisor";
|
||||
|
||||
# MACVTAP bridge to the host's network
|
||||
interfaces = [
|
||||
{
|
||||
type = "macvtap";
|
||||
id = "vm-${vmName}";
|
||||
macvtap = {
|
||||
link = vmCfg.macvtap;
|
||||
mode = "bridge";
|
||||
};
|
||||
inherit (vmCfg) mac;
|
||||
}
|
||||
];
|
||||
|
||||
shares =
|
||||
[
|
||||
# Share the nix-store of the host
|
||||
{
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "ro-store";
|
||||
proto = "virtiofs";
|
||||
}
|
||||
]
|
||||
# Mount persistent data from the host
|
||||
++ optionalAttrs vmCfg.zfs.enable {
|
||||
source = vmCfg.zfs.mountpoint;
|
||||
mountPoint = "/persist";
|
||||
tag = "persist";
|
||||
proto = "virtiofs";
|
||||
};
|
||||
};
|
||||
|
||||
fileSystems."/persist".neededForBoot = true;
|
||||
|
||||
# Add a writable store overlay, but since this is always ephemeral
|
||||
# disable any store optimization from nix.
|
||||
microvm.writableStoreOverlay = "/nix/.rw-store";
|
||||
nix = {
|
||||
settings.auto-optimise-store = mkForce false;
|
||||
optimise.automatic = mkForce false;
|
||||
gc.automatic = mkForce false;
|
||||
};
|
||||
|
||||
extra.networking.renameInterfacesByMac.${vmCfg.linkName} = vmCfg.mac;
|
||||
|
||||
systemd.network.networks = {
|
||||
"10-${vmCfg.linkName}" = {
|
||||
matchConfig.Name = vmCfg.linkName;
|
||||
DHCP = "yes";
|
||||
networkConfig = {
|
||||
IPv6PrivacyExtensions = "yes";
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
linkConfig.RequiredForOnline = "routable";
|
||||
};
|
||||
};
|
||||
|
||||
# TODO change once microvms are compatible with stage-1 systemd
|
||||
boot.initrd.systemd.enable = mkForce false;
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
imports = [microvm.host];
|
||||
imports = [
|
||||
# Add the host module, but only enable if it necessary
|
||||
microvm.host
|
||||
{microvm.host.enable = cfg != {};}
|
||||
];
|
||||
|
||||
options.extra.microvms = mkOption {
|
||||
default = {};
|
||||
description = "Provides a base configuration for MicroVMs.";
|
||||
description = "Handles the necessary base setup for MicroVMs.";
|
||||
type = types.attrsOf (types.submodule {
|
||||
options = {
|
||||
zfs = {
|
||||
enable = mkEnableOption (mdDoc "Enable persistent data on separate zfs dataset");
|
||||
|
||||
pool = mkOption {
|
||||
type = types.str;
|
||||
description = mdDoc "The host's zfs pool on which the dataset resides";
|
||||
};
|
||||
|
||||
dataset = mkOption {
|
||||
type = types.str;
|
||||
description = mdDoc "The host's dataset that should be used for this vm's state (will automatically be created, parent dataset must exist)";
|
||||
};
|
||||
|
||||
mountpoint = mkOption {
|
||||
type = types.str;
|
||||
description = mdDoc "The host's mountpoint for the vm's dataset (will be shared via virtofs as /persist in the vm)";
|
||||
};
|
||||
|
||||
disko = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = mdDoc "Add this dataset to the host's disko configuration";
|
||||
};
|
||||
};
|
||||
|
||||
autostart = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
|
|
@ -137,22 +211,8 @@ in {
|
|||
});
|
||||
};
|
||||
|
||||
config = {
|
||||
assertions = let
|
||||
duplicateMacs = extraLib.duplicates (mapAttrsToList (_: vmCfg: vmCfg.mac) cfg);
|
||||
in [
|
||||
{
|
||||
assertion = duplicateMacs == [];
|
||||
message = "Duplicate MicroVM MAC addresses: ${concatStringsSep ", " duplicateMacs}";
|
||||
}
|
||||
];
|
||||
|
||||
microvm = {
|
||||
host.enable = cfg != {};
|
||||
declarativeUpdates = true;
|
||||
restartIfChanged = true;
|
||||
vms = mkIf (cfg != {}) (mapAttrs defineMicrovm cfg);
|
||||
autostart = mkIf (cfg != {}) (attrNames (filterAttrs (_: v: v.autostart) cfg));
|
||||
};
|
||||
};
|
||||
config =
|
||||
mkIf (cfg != {})
|
||||
(extraLib.mkMergeTopLevel ["assertions" "disko" "systemd" "microvm"]
|
||||
([hostConfig] ++ mapAttrsToList microvmConfig cfg));
|
||||
}
|
||||
|
|
|
|||
11
nix/lib.nix
11
nix/lib.nix
|
|
@ -13,11 +13,15 @@
|
|||
escapeShellArg
|
||||
filter
|
||||
flatten
|
||||
foldAttrs
|
||||
foldl'
|
||||
genAttrs
|
||||
getAttrs
|
||||
head
|
||||
mapAttrs
|
||||
mapAttrs'
|
||||
mergeAttrs
|
||||
mkMerge
|
||||
nameValuePair
|
||||
optionalAttrs
|
||||
partition
|
||||
|
|
@ -49,6 +53,13 @@ in rec {
|
|||
# True if the path or string starts with /
|
||||
isAbsolutePath = x: substring 0 1 x == "/";
|
||||
|
||||
# Used to merge multiple toplevel configuration entries
|
||||
# https://gist.github.com/udf/4d9301bdc02ab38439fd64fbda06ea43
|
||||
mkMergeTopLevel = names: attrs:
|
||||
getAttrs names (
|
||||
mapAttrs (_: mkMerge) (foldAttrs (n: a: [n] ++ a) [] attrs)
|
||||
);
|
||||
|
||||
disko = {
|
||||
gpt = {
|
||||
partEfi = name: start: end: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue