feat: use declaratively provisioned influx tokens for telegraf

modules/default.nix

@@ -18,7 +18,6 @@
     ./config/users.nix
     ./config/xdg.nix
 
-    ./meta/influxdb-retrieve.nix
     ./meta/influxdb.nix
     ./meta/microvms.nix
     ./meta/nginx.nix

modules/meta/influxdb-retrieve.nix

@@ -1,33 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}: let
-  inherit
-    (lib)
-    mkOption
-    types
-    ;
-
-  cfg = config.services.influxdb2;
-in {
-  options.services.influxdb2.provision.retrieveToken = mkOption {
-    type = types.functionTo (types.functionTo types.str);
-    readOnly = true;
-    description = "Script that returns a agenix-rekey generator to retrieve the given token";
-    default = def: let
-      id = builtins.substring 0 32 (builtins.hashString "sha256" "${def.user}:${def.org}:${def.name}");
-    in
-      {
-        pkgs,
-        lib,
-        ...
-      }: ''
-        echo " -> Retrieving influxdb token ${def.name} for org ${def.org} on ${config.node.name}" >&2
-        ssh ${config.node.name} -- \
-          'bash -c '"'"'influx auth list --json --token "$(< ${cfg.provision.initialSetup.tokenFile})"'"'" \
-          | ${lib.getExe pkgs.jq} -r '.[] | select(.description | contains("${id}")) | .token' \
-          || die "Could not list/find influxdb api token '${def.name}' (${id})"
-      '';
-  };
-}

modules/meta/influxdb.nix

@@ -545,11 +545,14 @@ in {
       preStart = ''
         if ! test -e "$STATE_DIRECTORY/influxd.bolt"; then
           touch "$STATE_DIRECTORY/.first_startup"
+        else
+          # Manipulate provisioned api tokens if necessary
+          ${getExe tokenManipulator} "$STATE_DIRECTORY/influxd.bolt"
         fi
       '';
 
       postStart = let
-        influxCli = "${pkgs.influxdb2-cli}/bin/influx"; # getExe pkgs.influxdb2-cli
+        influxCli = getExe pkgs.influxdb2-cli;
       in
         ''
           set -euo pipefail
@@ -589,6 +592,7 @@ in {
           fi
 
           export INFLUX_TOKEN=$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})
+          any_tokens_created=0
         ''
         + flip concatMapStrings cfg.provision.deleteApiTokens (apiToken: ''
           if id=$(
@@ -839,11 +843,17 @@ in {
             "--write-bucket" "''${bucketIds[${escapeShellArg bucket}]}"
           '')}
             )
-            ${influxCli} auth create ${escapeShellArgs createArgs} >/dev/null \
-              "''${extraArgs[@]}"
+            ${influxCli} auth create ${escapeShellArgs createArgs} >/dev/null "''${extraArgs[@]}"
+            any_tokens_created=1
             echo "Created api token org="${escapeShellArg apiToken.org}" user="${escapeShellArg apiToken.user}
           fi
-        '');
+        '')
+        + ''
+          if [[ $any_tokens_created == 1 ]]; then
+            echo "Created new tokens, forcing service restart so we can manipulate secrets"
+            kill "$MAINPID"
+          fi
+        '';
     };
   };
 }

modules/meta/telegraf.nix

@@ -7,6 +7,7 @@
 }: let
   inherit
     (lib)
+    mkAfter
     mkEnableOption
     mkIf
     mkOption
@@ -55,141 +56,137 @@ in {
     };
   };
 
-  config = let
-    tokenDef = {
-      name = "telegraf (${config.node.name})";
-      org = "servers";
-      user = "admin";
-      readBuckets = ["telegraf"];
-      writeBuckets = ["telegraf"];
+  config = mkIf cfg.enable {
+    nodes.${cfg.influxdb2.node}.services.influxdb2.provision.ensureApiTokens = [
+      {
+        name = "telegraf (${config.node.name})";
+        org = "servers";
+        user = "admin";
+        readBuckets = ["telegraf"];
+        writeBuckets = ["telegraf"];
+        tokenFile = config.age.secrets.telegraf-influxdb-token.path;
+      }
+    ];
+
+    age.secrets.telegraf-influxdb-token = {
+      generator.script = "alnum";
+      generator.tags = ["influxdb"];
+      mode = "440";
+      group = "telegraf";
     };
-  in
-    mkIf cfg.enable {
-      nodes.${cfg.influxdb2.node}.services.influxdb2.provision.ensureApiTokens = [
-        tokenDef
+
+    security.elewrap.telegraf-sensors = mkIf cfg.scrapeSensors {
+      command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"];
+      targetUser = "root";
+      allowedUsers = ["telegraf"];
+    };
+
+    security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable {
+      command = ["${pkgs.nvme-cli}/bin/nvme"];
+      targetUser = "root";
+      allowedUsers = ["telegraf"];
+      passArguments = true;
+    };
+
+    security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable {
+      command = ["${pkgs.smartmontools}/bin/smartctl"];
+      targetUser = "root";
+      allowedUsers = ["telegraf"];
+      passArguments = true;
+    };
+
+    services.telegraf = {
+      enable = true;
+      environmentFiles = ["/run/telegraf/env"];
+      extraConfig = {
+        agent = {
+          interval = "10s";
+          round_interval = true; # Always collect on :00,:10,...
+          metric_batch_size = 5000;
+          metric_buffer_limit = 50000;
+          collection_jitter = "0s";
+          flush_interval = "20s";
+          flush_jitter = "5s";
+          precision = "1ms";
+          hostname = config.node.name;
+          omit_hostname = false;
+        };
+        outputs = {
+          influxdb_v2 = {
+            urls = ["https://${cfg.influxdb2.domain}"];
+            token = "$INFLUX_TOKEN";
+            inherit (cfg.influxdb2) organization bucket;
+          };
+        };
+        inputs =
+          {
+            conntrack = {};
+            cpu = {};
+            disk = {};
+            diskio = {};
+            internal = {};
+            interrupts = {};
+            kernel = {};
+            kernel_vmstat = {};
+            linux_sysctl_fs = {};
+            mem = {};
+            net = {};
+            netstat = {};
+            nstat = {};
+            processes = {};
+            swap = {};
+            system = {};
+            systemd_units = {
+              unittype = "service";
+            };
+            temp = {};
+            wireguard = {};
+            # http_response = { urls = [ "http://localhost/" ]; };
+            # ping = { urls = [ "9.9.9.9" ]; };
+          }
+          // optionalAttrs config.services.smartd.enable {
+            sensors = {};
+            smart = {
+              attributes = true;
+              path_nvme = config.security.elewrap.telegraf-nvme.path;
+              path_smartctl = config.security.elewrap.telegraf-smartctl.path;
+              use_sudo = false;
+            };
+          }
+          // optionalAttrs config.services.nginx.enable {
+            nginx.urls = ["http://localhost/nginx_status"];
+          }
+          // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
+            wireless = {};
+          };
+      };
+    };
+
+    services.nginx.virtualHosts = mkIf config.services.nginx.enable {
+      localhost.listenAddresses = ["127.0.0.1" "[::1]"];
+      localhost.locations."= /nginx_status".extraConfig = ''
+        allow 127.0.0.0/8;
+        allow ::1;
+        deny all;
+        stub_status;
+        access_log off;
+      '';
+    };
+
+    systemd.services.telegraf = {
+      path = [
+        # Make sensors refer to the correct wrapper
+        (mkIf cfg.scrapeSensors
+          (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path))
       ];
-
-      age.secrets.telegraf-influxdb-token = {
-        generator = {
-          script = args: ''
-            echo -n "INFLUX_TOKEN="
-            ${nodes.${cfg.influxdb2.node}.config.services.influxdb2.provision.retrieveToken tokenDef args}
-          '';
-          tags = ["influxdb"];
-        };
-        mode = "440";
-        group = "telegraf";
-      };
-
-      security.elewrap.telegraf-sensors = mkIf cfg.scrapeSensors {
-        command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"];
-        targetUser = "root";
-        allowedUsers = ["telegraf"];
-      };
-
-      security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable {
-        command = ["${pkgs.nvme-cli}/bin/nvme"];
-        targetUser = "root";
-        allowedUsers = ["telegraf"];
-        passArguments = true;
-      };
-
-      security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable {
-        command = ["${pkgs.smartmontools}/bin/smartctl"];
-        targetUser = "root";
-        allowedUsers = ["telegraf"];
-        passArguments = true;
-      };
-
-      services.telegraf = {
-        enable = true;
-        environmentFiles = [config.age.secrets.telegraf-influxdb-token.path];
-        extraConfig = {
-          agent = {
-            interval = "10s";
-            round_interval = true; # Always collect on :00,:10,...
-            metric_batch_size = 5000;
-            metric_buffer_limit = 50000;
-            collection_jitter = "0s";
-            flush_interval = "20s";
-            flush_jitter = "5s";
-            precision = "1ms";
-            hostname = config.node.name;
-            omit_hostname = false;
-          };
-          outputs = {
-            influxdb_v2 = {
-              urls = ["https://${cfg.influxdb2.domain}"];
-              token = "$INFLUX_TOKEN";
-              inherit (cfg.influxdb2) organization bucket;
-            };
-          };
-          inputs =
-            {
-              conntrack = {};
-              cpu = {};
-              disk = {};
-              diskio = {};
-              internal = {};
-              interrupts = {};
-              kernel = {};
-              kernel_vmstat = {};
-              linux_sysctl_fs = {};
-              mem = {};
-              net = {};
-              netstat = {};
-              nstat = {};
-              processes = {};
-              swap = {};
-              system = {};
-              systemd_units = {
-                unittype = "service";
-              };
-              temp = {};
-              wireguard = {};
-              # http_response = { urls = [ "http://localhost/" ]; };
-              # ping = { urls = [ "9.9.9.9" ]; };
-            }
-            // optionalAttrs config.services.smartd.enable {
-              sensors = {};
-              smart = {
-                attributes = true;
-                path_nvme = config.security.elewrap.telegraf-nvme.path;
-                path_smartctl = config.security.elewrap.telegraf-smartctl.path;
-                use_sudo = false;
-              };
-            }
-            // optionalAttrs config.services.nginx.enable {
-              nginx.urls = ["http://localhost/nginx_status"];
-            }
-            // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
-              wireless = {};
-            };
-        };
-      };
-
-      services.nginx.virtualHosts = mkIf config.services.nginx.enable {
-        localhost.listenAddresses = ["127.0.0.1" "[::1]"];
-        localhost.locations."= /nginx_status".extraConfig = ''
-          allow 127.0.0.0/8;
-          allow ::1;
-          deny all;
-          stub_status;
-          access_log off;
-        '';
-      };
-
-      systemd.services.telegraf = {
-        path = [
-          # Make sensors refer to the correct wrapper
-          (mkIf cfg.scrapeSensors
-            (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path))
-        ];
-        serviceConfig = {
-          # For wireguard statistics
-          AmbientCapabilities = ["CAP_NET_ADMIN"];
-          RestartSec = "600"; # Retry every 10 minutes
-        };
+      preStart = mkAfter ''
+        echo "INFLUX_TOKEN=$(< ${config.age.secrets.telegraf-influxdb-token.path})" > /run/telegraf/env
+      '';
+      serviceConfig = {
+        # For wireguard statistics
+        AmbientCapabilities = ["CAP_NET_ADMIN"];
+        RestartSec = "600"; # Retry every 10 minutes
       };
     };
+  };
 }