From 9833fd50ce2cedcda35d99e7772e26bd9be1ec54 Mon Sep 17 00:00:00 2001 From: oddlama Date: Sun, 17 Dec 2023 17:16:12 +0100 Subject: [PATCH] feat: use own nixosSystem invocation for containers, add bind mounts --- modules/config/secrets.nix | 6 ++- modules/default.nix | 1 + modules/guests/container.nix | 77 ++++++++++++++++++++++++++---------- modules/guests/default.nix | 13 ++++++ modules/guests/microvm.nix | 3 +- nix/hosts.nix | 20 +++++----- 6 files changed, 86 insertions(+), 34 deletions(-) diff --git a/modules/config/secrets.nix b/modules/config/secrets.nix index 81bded7..26fc11d 100644 --- a/modules/config/secrets.nix +++ b/modules/config/secrets.nix @@ -54,6 +54,8 @@ # to create a link called /run/agenix. Agenix should probably fail in this case, # but doesn't and instead puts the generation link into the existing directory. # TODO See https://github.com/ryantm/agenix/pull/187. - system.activationScripts.removeAgenixLink.text = "[[ ! -L /run/agenix ]] && [[ -d /run/agenix ]] && rm -rf /run/agenix"; - system.activationScripts.agenixNewGeneration.deps = ["removeAgenixLink"]; + system.activationScripts = lib.mkIf (config.age.secrets != {}) { + removeAgenixLink.text = "[[ ! -L /run/agenix ]] && [[ -d /run/agenix ]] && rm -rf /run/agenix"; + activationScripts.agenixNewGeneration.deps = ["removeAgenixLink"]; + }; } diff --git a/modules/default.nix b/modules/default.nix index 37bfbbc..9d6d277 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -27,6 +27,7 @@ ./guests + # TODO merge as ./* ./meta/kanidm.nix ./meta/nginx.nix ./meta/oauth2-proxy.nix diff --git a/modules/guests/container.nix b/modules/guests/container.nix index 8649b7a..a0558e2 100644 --- a/modules/guests/container.nix +++ b/modules/guests/container.nix @@ -1,29 +1,66 @@ guestName: guestCfg: { config, + inputs, lib, + minimal, + nodes, + pkgs, ... -} @ attrs: let - inherit (lib) mkMerge; -in { +}: { autoStart = guestCfg.autostart; - specialArgs = - attrs - // { - parentNode = config; - }; - macvlans = [guestCfg.container.macvlan]; + macvlans = ["${guestCfg.container.macvlan}:${guestCfg.networking.mainLinkName}"]; ephemeral = true; privateNetwork = true; - config = mkMerge (guestCfg.modules - ++ [ - (import ./common-guest-config.nix guestName guestCfg) - { - systemd.network.networks = { - "10-${guestCfg.networking.mainLinkName}" = { - matchConfig.OriginalName = "mv-${guestCfg.container.macvlan}"; - linkConfig.Name = guestCfg.networking.mainLinkName; + # We bind-mount stuff from the host into /guest first, and later bind + # mount them into the correct path inside the guest, so we have a + # fileSystems entry that impermanence can depend upon. + bindMounts = { + "/guest/state" = { + hostPath = "/state/guests/${guestName}"; + isReadOnly = false; + }; + # Mount persistent data from the host + "/guest/persist" = lib.mkIf guestCfg.zfs.enable { + hostPath = guestCfg.zfs.mountpoint; + isReadOnly = false; + }; + }; + nixosConfiguration = inputs.nixpkgs.lib.nixosSystem { + specialArgs = { + inherit lib nodes inputs minimal; + }; + prefix = ["nodes" "${config.node.name}-${guestName}" "config"]; + system = null; + modules = + [ + { + boot.isContainer = true; + networking.useHostResolvConf = false; + + # We cannot force the package set via nixpkgs.pkgs and + # inputs.nixpkgs.nixosModules.readOnlyPkgs, since some nixosModules + # like nixseparatedebuginfod depend on adding packages via nixpkgs.overlays. + # So we just mimic the options and overlays defined by the passed pkgs set. + nixpkgs.hostPlatform = config.nixpkgs.hostPlatform.system; + nixpkgs.overlays = pkgs.overlays; + nixpkgs.config = pkgs.config; + + # Bind the /guest/* paths from above so impermancence doesn't complain. + fileSystems."/state" = { + fsType = "none"; + neededForBoot = true; + device = "/guest/state"; + options = ["bind"]; }; - }; - } - ]); + fileSystems."/persist" = lib.mkIf guestCfg.zfs.enable { + fsType = "none"; + neededForBoot = true; + device = "/guest/persist"; + options = ["bind"]; + }; + } + (import ./common-guest-config.nix guestName guestCfg) + ] + ++ guestCfg.modules; + }; } diff --git a/modules/guests/default.nix b/modules/guests/default.nix index fe8a43d..2381154 100644 --- a/modules/guests/default.nix +++ b/modules/guests/default.nix @@ -103,6 +103,19 @@ in { default = "host"; }; + options.containers = mkOption { + type = types.attrsOf (types.submodule (submod: { + options.nixosConfiguration = mkOption { + type = types.unspecified; + default = null; + description = "Set this to the result of a `nixosSystem` invocation to use it as the guest system. This will set the `path` option for you."; + }; + config = mkIf (submod.config.nixosConfiguration != null) { + path = submod.config.nixosConfiguration.config.system.build.toplevel; + }; + })); + }; + options.guests = mkOption { default = {}; description = "Defines the actual vms and handles the necessary base setup for them."; diff --git a/modules/guests/microvm.nix b/modules/guests/microvm.nix index d585c9d..518e84d 100644 --- a/modules/guests/microvm.nix +++ b/modules/guests/microvm.nix @@ -18,7 +18,6 @@ guestName: guestCfg: { mac = (net.mac.assignMacs "02:01:27:00:00:00" 24 [] (attrNames config.guests)).${guestName}; in { specialArgs = { - parentNode = config; inherit (inputs.self) nodes; inherit (inputs.self.pkgs.${guestCfg.microvm.system}) lib; inherit inputs; @@ -77,7 +76,7 @@ in { }; }; - # FIXME this should be changed in microvm.nix to mkDefault in oder to not require mkForce here + # FIXME this should be changed in microvm.nix to mkDefault in order to not require mkForce here fileSystems."/state".neededForBoot = mkForce true; fileSystems."/persist".neededForBoot = mkForce true; diff --git a/nix/hosts.nix b/nix/hosts.nix index abd5e2a..de69f85 100644 --- a/nix/hosts.nix +++ b/nix/hosts.nix @@ -52,16 +52,16 @@ inputs: let # together with it. We collect all defined guests from each node here # to allow accessing any node via the unified attribute `nodes`. guestConfigs = flip concatMapAttrs self.nixosConfigurations (_: node: - flip mapAttrs' (node.config.guests or {}) (guestName: guestDef: - nameValuePair guestDef.nodeName ( - if guestDef.backend == "microvm" - then node.config.microvm.vms.${guestName}.config - else { - # We can only access the .config part of nixosSystem here unfortunately, - # since the rest is not exposed by the nixos module. - inherit (node.config.containers.${guestName}) config; - } - ))); + flip mapAttrs' (node.config.guests or {}) ( + guestName: guestDef: + nameValuePair guestDef.nodeName + ( + if guestDef.backend == "microvm" + then node.config.microvm.vms.${guestName}.config + else node.config.containers.${guestName} + ) + .nixosConfiguration + )); in { inherit hosts