feat: automatically provision tokens for telegraf

hosts/nom/default.nix

@@ -32,113 +32,4 @@
     font = "ter-v28n";
     packages = [pkgs.terminus_font];
   };
-
-  services.influxdb2 = {
-    enable = true;
-    settings = {
-      reporting-disabled = true;
-      http-bind-address = "localhost:8086";
-    };
-    initialSetup = {
-      enable = true;
-      organization = "servers";
-      bucket = "telegraf";
-
-      passwordFile = pkgs.writeText "tmp-pw" "ExAmPl3PA55W0rD";
-      tokenFile = pkgs.writeText "tmp-tok" "asroiuhoiuahnawo4unhasdorviuhngoiuhraoug";
-    };
-    deleteOrganizations = ["delorg"];
-    deleteBuckets = [
-      {
-        name = "delbucket";
-        org = "delorg";
-      }
-    ];
-    deleteUsers = ["deluser"];
-    deleteRemotes = [
-      {
-        name = "delremote";
-        org = "delorg";
-      }
-    ];
-    deleteReplications = [
-      {
-        name = "delreplication";
-        org = "delorg";
-      }
-    ];
-    deleteApiTokens = [
-      {
-        name = "deltoken";
-        org = "delorg";
-        user = "deluser";
-      }
-    ];
-    ensureOrganizations = [
-      {
-        name = "myorg";
-        description = "Myorg description";
-      }
-      #{
-      #  name = "delorg";
-      #}
-    ];
-    ensureBuckets = [
-      {
-        name = "mybucket";
-        org = "myorg";
-        description = "Mybucket description";
-      }
-      #{
-      #  name = "delbucket";
-      #  org = "delorg";
-      #}
-    ];
-    ensureUsers = [
-      {
-        name = "myuser";
-        org = "myorg";
-        passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
-      }
-      #{
-      #  name = "deluser";
-      #  org = "delorg";
-      #  passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
-      #}
-    ];
-    #ensureRemotes = [
-    #  {
-    #    name = "delremote";
-    #    org = "delorg";
-    #    remoteUrl = "http://localhost:8087";
-    #    remoteOrgId = "a1b2c3d4a1b2c3d4";
-    #    remoteTokenFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
-    #  }
-    #];
-    #ensureReplications = [
-    #  {
-    #    name = "delreplication";
-    #    org = "delorg";
-    #    remote = "delremote";
-    #    localBucket = "delbucket";
-    #    remoteBucket = "delbucket2";
-    #  }
-    #];
-    ensureApiTokens = [
-      {
-        name = "mytoken";
-        org = "myorg";
-        user = "myuser";
-        readBuckets = ["mybucket"];
-        writeBuckets = ["mybucket"];
-      }
-      #{
-      #  name = "deltoken";
-      #  org = "delorg";
-      #  user = "deluser";
-      #  readBuckets = ["delbucket"];
-      #  writeBuckets = ["delbucket"];
-      #}
-    ];
-  };
 }

hosts/sentinel/default.nix

@@ -29,8 +29,12 @@
   networking.hosts.${config.meta.wireguard.proxy-sentinel.ipv4} = [config.networking.providedDomains.influxdb];
   meta.telegraf = {
     enable = true;
-    influxdb2.domain = config.networking.providedDomains.influxdb;
+    scrapeSensors = false;
-    influxdb2.organization = "servers";
+    influxdb2 = {
-    influxdb2.bucket = "telegraf";
+      domain = config.networking.providedDomains.influxdb;
+      organization = "servers";
+      bucket = "telegraf";
+      node = "ward-influxdb";
+    };
   };
 }

hosts/ward/default.nix

@@ -32,9 +32,12 @@
   networking.hosts.${nodes.sentinel.config.meta.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.networking.providedDomains.influxdb];
   meta.telegraf = {
     enable = true;
-    influxdb2.domain = nodes.sentinel.config.networking.providedDomains.influxdb;
+    influxdb2 = {
-    influxdb2.organization = "servers";
+      domain = nodes.sentinel.config.networking.providedDomains.influxdb;
-    influxdb2.bucket = "telegraf";
+      organization = "servers";
+      bucket = "telegraf";
+      node = "ward-influxdb";
+    };
   };
 
   # TODO track my github stats

hosts/ward/microvms/common.nix

@@ -11,8 +11,12 @@ in {
   networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb];
   meta.telegraf = {
     enable = true;
-    influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb;
+    scrapeSensors = false;
-    influxdb2.organization = "servers";
+    influxdb2 = {
-    influxdb2.bucket = "telegraf";
+      domain = sentinelCfg.networking.providedDomains.influxdb;
+      organization = "servers";
+      bucket = "telegraf";
+      node = "ward-influxdb";
+    };
   };
 }

hosts/ward/microvms/influxdb.nix

@@ -3,6 +3,7 @@
   lib,
   nodes,
   utils,
+  pkgs,
   ...
 }: let
   sentinelCfg = nodes.sentinel.config;
@@ -51,14 +52,54 @@ in {
     };
   };
 
+  age.secrets.influxdb-admin-password = {
+    generator.script = "alnum";
+    mode = "440";
+    group = "influxdb2";
+  };
+
+  age.secrets.influxdb-admin-token = {
+    generator.script = "alnum";
+    mode = "440";
+    group = "influxdb2";
+  };
+
+  age.secrets.influxdb-user-telegraf-token = {
+    generator.script = "alnum";
+    mode = "440";
+    group = "influxdb2";
+  };
+
   services.influxdb2 = {
     enable = true;
     settings = {
       reporting-disabled = true;
       http-bind-address = "${config.meta.wireguard.proxy-sentinel.ipv4}:${toString influxdbPort}";
     };
+    provision = {
+      enable = true;
+      initialSetup = {
+        organization = "default";
+        bucket = "default";
+        passwordFile = config.age.secrets.influxdb-admin-password.path;
+        tokenFile = config.age.secrets.influxdb-admin-token.path;
+      };
+      ensureOrganizations = [
+        {
+          name = "servers";
+        }
+      ];
+      ensureBuckets = [
+        {
+          name = "telegraf";
+          org = "servers";
+        }
+      ];
+    };
   };
 
+  environment.systemPackages = [pkgs.influxdb2-cli];
+
   systemd.services.influxdb2 = {
     after = ["sys-subsystem-net-devices-${utils.escapeSystemdPath "proxy-sentinel"}.device"];
     serviceConfig.RestartSec = "600"; # Retry every 10 minutes

hosts/zackbiene/default.nix

@@ -47,9 +47,12 @@ in {
   networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb];
   meta.telegraf = {
     enable = true;
-    influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb;
+    influxdb2 = {
-    influxdb2.organization = "servers";
+      domain = sentinelCfg.networking.providedDomains.influxdb;
-    influxdb2.bucket = "telegraf";
+      organization = "servers";
+      bucket = "telegraf";
+      node = "ward-influxdb";
+    };
   };
 
   # Fails if there are no SMART devices

modules/default.nix

@@ -18,6 +18,7 @@
     ./config/users.nix
     ./config/xdg.nix
 
+    ./meta/influxdb-retrieve.nix
     ./meta/influxdb.nix
     ./meta/microvms.nix
     ./meta/nginx.nix

modules/meta/influxdb-retrieve.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit
+    (lib)
+    mkOption
+    types
+    ;
+
+  cfg = config.services.influxdb2;
+in {
+  options.services.influxdb2.provision.retrieveToken = mkOption {
+    type = types.functionTo (types.functionTo types.str);
+    readOnly = true;
+    description = "Script that returns a agenix-rekey generator to retrieve the given token";
+    default = def: let
+      id = builtins.substring 0 32 (builtins.hashString "sha256" "${def.user}:${def.org}:${def.name}");
+    in
+      {
+        pkgs,
+        lib,
+        ...
+      }: ''
+        echo " -> Retrieving influxdb token ${def.name} for org ${def.org} on ${config.node.name}" >&2
+        ssh ${config.node.name} -- \
+          'bash -c '"'"'influx auth list --json --token "$(< ${cfg.provision.initialSetup.tokenFile})"'"'" \
+          | ${lib.getExe pkgs.jq} -r '.[] | select(.description | contains("${id}")) | .token' \
+          || die "Could not list/find influxdb api token '${def.name}' (${id})"
+      '';
+  };
+}

modules/meta/influxdb.nix

@@ -31,9 +31,9 @@
 
   cfg = config.services.influxdb2;
 in {
-  options.services.influxdb2 = {
+  options.services.influxdb2.provision = {
+    enable = mkEnableOption "initial database setup";
     initialSetup = {
-      enable = mkEnableOption "initial database setup";
       organization = mkOption {
         type = types.str;
         example = "main";
@@ -408,7 +408,7 @@ in {
     };
   };
 
-  config = mkIf (cfg.enable && cfg.initialSetup.enable) {
+  config = mkIf (cfg.enable && cfg.provision.enable) {
     assertions = let
       validPermissions = flip genAttrs (x: true) [
         "authorizations"
@@ -435,23 +435,23 @@ in {
         "replications"
       ];
 
-      knownOrgs = map (x: x.name) cfg.ensureOrganizations;
+      knownOrgs = map (x: x.name) cfg.provision.ensureOrganizations;
-      knownRemotes = map (x: x.name) cfg.ensureRemotes;
+      knownRemotes = map (x: x.name) cfg.provision.ensureRemotes;
-      knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.ensureBuckets);
+      knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.provision.ensureBuckets);
     in
-      flip concatMap cfg.ensureBuckets (bucket: [
+      flip concatMap cfg.provision.ensureBuckets (bucket: [
         {
           assertion = elem bucket.org knownOrgs;
           message = "The influxdb bucket '${bucket.name}' refers to an unknown organization '${bucket.org}'.";
         }
       ])
-      ++ flip concatMap cfg.ensureUsers (user: [
+      ++ flip concatMap cfg.provision.ensureUsers (user: [
         {
           assertion = elem user.org knownOrgs;
           message = "The influxdb user '${user.name}' refers to an unknown organization '${user.org}'.";
         }
       ])
-      ++ flip concatMap cfg.ensureRemotes (remote: [
+      ++ flip concatMap cfg.provision.ensureRemotes (remote: [
         {
           assertion = (remote.remoteOrgId == null) != (remote.remoteOrg == null);
           message = "The influxdb remote '${remote.name}' must specify exactly one of remoteOrgId or remoteOrg.";
@@ -461,19 +461,19 @@ in {
           message = "The influxdb remote '${remote.name}' refers to an unknown organization '${remote.org}'.";
         }
       ])
-      ++ flip concatMap cfg.ensureReplications (replication: [
+      ++ flip concatMap cfg.provision.ensureReplications (replication: [
         {
           assertion = elem replication.remote knownRemotes;
           message = "The influxdb replication '${replication.name}' refers to an unknown remote '${replication.remote}'.";
         }
         (let
-          remote = head (filter (x: x.name == replication.remote) cfg.ensureRemotes);
+          remote = head (filter (x: x.name == replication.remote) cfg.provision.ensureRemotes);
         in {
           assertion = elem replication.localBucket (knownBucketsFor remote.org);
           message = "The influxdb replication '${replication.name}' refers to an unknown bucket '${replication.localBucket}' in organization '${remote.org}'.";
         })
       ])
-      ++ flip concatMap cfg.ensureApiTokens (apiToken: let
+      ++ flip concatMap cfg.provision.ensureApiTokens (apiToken: let
         validBuckets = flip genAttrs (x: true) (knownBucketsFor apiToken.org);
       in [
         {
@@ -557,20 +557,20 @@ in {
             # avoid saving the token as the active config.
             ${influxCli} setup \
               --configs-path /dev/null \
-              --org ${escapeShellArg cfg.initialSetup.organization} \
+              --org ${escapeShellArg cfg.provision.initialSetup.organization} \
-              --bucket ${escapeShellArg cfg.initialSetup.bucket} \
+              --bucket ${escapeShellArg cfg.provision.initialSetup.bucket} \
-              --username ${escapeShellArg cfg.initialSetup.username} \
+              --username ${escapeShellArg cfg.provision.initialSetup.username} \
-              --password "$(< ${escapeShellArg cfg.initialSetup.passwordFile})" \
+              --password "$(< ${escapeShellArg cfg.provision.initialSetup.passwordFile})" \
-              --token "$(< ${escapeShellArg cfg.initialSetup.tokenFile})" \
+              --token "$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})" \
-              --retention ${escapeShellArg cfg.initialSetup.retention} \
+              --retention ${escapeShellArg cfg.provision.initialSetup.retention} \
               --force >/dev/null
 
             rm -f "$STATE_DIRECTORY/.first_startup"
           fi
 
-          export INFLUX_TOKEN=$(< ${escapeShellArg cfg.initialSetup.tokenFile})
+          export INFLUX_TOKEN=$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})
         ''
-        + flip concatMapStrings cfg.deleteApiTokens (apiToken: ''
+        + flip concatMapStrings cfg.provision.deleteApiTokens (apiToken: ''
           if id=$(
             ${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \
               | ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id'
@@ -579,7 +579,7 @@ in {
             echo "Deleted api token id="${escapeShellArg apiToken.id}
           fi
         '')
-        + flip concatMapStrings cfg.deleteReplications (replication: ''
+        + flip concatMapStrings cfg.provision.deleteReplications (replication: ''
           if id=$(
             ${influxCli} replication list --json --org ${escapeShellArg replication.org} --name ${escapeShellArg replication.name} 2>/dev/null \
               | ${getExe pkgs.jq} -r ".[0].id"
@@ -588,7 +588,7 @@ in {
             echo "Deleted replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name}
           fi
         '')
-        + flip concatMapStrings cfg.deleteRemotes (remote: ''
+        + flip concatMapStrings cfg.provision.deleteRemotes (remote: ''
           if id=$(
             ${influxCli} remote list --json --org ${escapeShellArg remote.org} --name ${escapeShellArg remote.name} 2>/dev/null \
               | ${getExe pkgs.jq} -r ".[0].id"
@@ -597,7 +597,7 @@ in {
             echo "Deleted remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name}
           fi
         '')
-        + flip concatMapStrings cfg.deleteUsers (user: ''
+        + flip concatMapStrings cfg.provision.deleteUsers (user: ''
           if id=$(
             ${influxCli} user list --json --name ${escapeShellArg user} 2>/dev/null \
               | ${getExe pkgs.jq} -r ".[0].id"
@@ -606,7 +606,7 @@ in {
             echo "Deleted user name="${escapeShellArg user}
           fi
         '')
-        + flip concatMapStrings cfg.deleteBuckets (bucket: ''
+        + flip concatMapStrings cfg.provision.deleteBuckets (bucket: ''
           if id=$(
             ${influxCli} bucket list --json --org ${escapeShellArg bucket.org} --name ${escapeShellArg bucket.name} 2>/dev/null \
               | ${getExe pkgs.jq} -r ".[0].id"
@@ -615,7 +615,7 @@ in {
             echo "Deleted bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name}
           fi
         '')
-        + flip concatMapStrings cfg.deleteOrganizations (org: ''
+        + flip concatMapStrings cfg.provision.deleteOrganizations (org: ''
           if id=$(
             ${influxCli} org list --json --name ${escapeShellArg org} 2>/dev/null \
               | ${getExe pkgs.jq} -r ".[0].id"
@@ -624,7 +624,7 @@ in {
             echo "Deleted org name="${escapeShellArg org}
           fi
         '')
-        + flip concatMapStrings cfg.ensureOrganizations (org: let
+        + flip concatMapStrings cfg.provision.ensureOrganizations (org: let
           listArgs = [
             "--name"
             org.name
@@ -645,7 +645,7 @@ in {
             echo "Created org name="${escapeShellArg org.name}
           fi
         '')
-        + flip concatMapStrings cfg.ensureBuckets (bucket: let
+        + flip concatMapStrings cfg.provision.ensureBuckets (bucket: let
           listArgs = [
             "--org"
             bucket.org
@@ -673,7 +673,7 @@ in {
             echo "Created bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name}
           fi
         '')
-        + flip concatMapStrings cfg.ensureUsers (user: let
+        + flip concatMapStrings cfg.provision.ensureUsers (user: let
           listArgs = [
             "--name"
             user.name
@@ -700,7 +700,7 @@ in {
             ${influxCli} user password ${escapeShellArgs listArgs} \
               --password "$(< ${escapeShellArg user.passwordFile})" >/dev/null
           '')
-        + flip concatMapStrings cfg.ensureRemotes (remote: let
+        + flip concatMapStrings cfg.provision.ensureRemotes (remote: let
           listArgs = [
             "--name"
             remote.name
@@ -746,7 +746,7 @@ in {
             echo "Created remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name}
           fi
         '')
-        + flip concatMapStrings cfg.ensureReplications (replication: let
+        + flip concatMapStrings cfg.provision.ensureReplications (replication: let
           listArgs = [
             "--name"
             replication.name
@@ -780,7 +780,7 @@ in {
             echo "Created replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name}
           fi
         '')
-        + flip concatMapStrings cfg.ensureApiTokens (apiToken: let
+        + flip concatMapStrings cfg.provision.ensureApiTokens (apiToken: let
           listArgs = [
             "--user"
             apiToken.user
@@ -801,7 +801,7 @@ in {
           if id=$(
             ${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \
               | ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id'
-          ); then
+          ) && [[ -n "$id" ]]; then
             true # No updateable args
           else
             declare -A bucketIds

modules/meta/telegraf.nix

@@ -7,7 +7,6 @@
 }: let
   inherit
     (lib)
-    mdDoc
     mkEnableOption
     mkIf
     mkOption
@@ -18,155 +17,179 @@
   cfg = config.meta.telegraf;
 in {
   options.meta.telegraf = {
-    enable = mkEnableOption (mdDoc "telegraf to push metrics to influx.");
+    enable = mkEnableOption "telegraf to push metrics to influx.";
+
+    scrapeSensors = mkOption {
+      type = types.bool;
+      default = true;
+      description = "Scrape sensors with lm_sensors. You should disable this for virtualized hosts.";
+    };
+
     influxdb2 = {
       domain = mkOption {
         type = types.str;
         example = "influxdb.example.com";
-        description = mdDoc "The influxdb v2 database to push to. https will be enforced.";
+        description = "The influxdb v2 database to push to. https will be enforced.";
       };
 
       organization = mkOption {
         type = types.str;
-        description = mdDoc "The organization to push to.";
+        description = "The organization to push to.";
       };
 
       bucket = mkOption {
         type = types.str;
-        description = mdDoc "The bucket to push to.";
+        description = "The bucket to push to.";
+      };
+
+      user = mkOption {
+        type = types.str;
+        default = "admin";
+        description = "The user for which the api key should be created.";
+      };
+
+      node = mkOption {
+        type = types.str;
+        description = "The node which hosts the influxdb service (used to provision an api token).";
       };
     };
   };
 
-  config = mkIf cfg.enable {
+  config = let
-    age.secrets.telegraf-influxdb-token = {
+    tokenDef = {
-      rekeyFile = config.node.secretsDir + "/telegraf-influxdb-token.age";
+      name = "telegraf (${config.node.name})";
-      # TODO generator.script = { pkgs, lib, decrypt, deps, ... }: let
+      org = "servers";
-      # TODO   adminBasicAuth = (builtins.head deps).file;
+      user = "admin";
-      # TODO   adminToken = (builtins.head deps).file; # TODO ..... filter by name?
+      readBuckets = ["telegraf"];
-      # TODO in ''
+      writeBuckets = ["telegraf"];
-      # TODO   echo " -> Provisioning influxdb token for telegraf on ${nodeName} at https://${cfg.influxdb2.domain}" >&2
-      # TODO   ${decrypt} ${lib.escapeShellArg aba.file} \
-      # TODO   INFLUX_HOST=https://${aba.host}+${aba.name}:${PW}@${URL}
-      # TODO     | ${pkgs.influxdb2-cli}/bin/influx -niBC 12 ${lib.escapeShellArg host}"+"${lib.escapeShellArg name} \
-      # TODO     || die "Failure"
-      # TODO '');
-      mode = "440";
-      group = "telegraf";
     };
-
+  in
-    # It's intentional to gate this behind smartd. Hosts without smartd are usually
+    mkIf cfg.enable {
-    # virtual and won't need sensor collection either.
+      nodes.${cfg.influxdb2.node}.services.influxdb2.provision.ensureApiTokens = [
-    # XXX: maybe as separate option?
+        tokenDef
-    security.elewrap.telegraf-sensors = mkIf config.services.smartd.enable {
-      command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"];
-      targetUser = "root";
-      allowedUsers = ["telegraf"];
-    };
-
-    security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable {
-      command = ["${pkgs.nvme-cli}/bin/nvme"];
-      targetUser = "root";
-      allowedUsers = ["telegraf"];
-      passArguments = true;
-    };
-
-    security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable {
-      command = ["${pkgs.smartmontools}/bin/smartctl"];
-      targetUser = "root";
-      allowedUsers = ["telegraf"];
-      passArguments = true;
-    };
-
-    services.telegraf = {
-      enable = true;
-      environmentFiles = [config.age.secrets.telegraf-influxdb-token.path];
-      extraConfig = {
-        agent = {
-          interval = "10s";
-          round_interval = true; # Always collect on :00,:10,...
-          metric_batch_size = 5000;
-          metric_buffer_limit = 50000;
-          collection_jitter = "0s";
-          flush_interval = "20s";
-          flush_jitter = "5s";
-          precision = "1ms";
-          hostname = config.node.name;
-          omit_hostname = false;
-        };
-        outputs = {
-          influxdb_v2 = {
-            urls = ["https://${cfg.influxdb2.domain}"];
-            token = "$INFLUX_TOKEN";
-            inherit (cfg.influxdb2) organization bucket;
-          };
-        };
-        inputs =
-          {
-            conntrack = {};
-            cpu = {};
-            disk = {};
-            diskio = {};
-            internal = {};
-            interrupts = {};
-            kernel = {};
-            kernel_vmstat = {};
-            linux_sysctl_fs = {};
-            mem = {};
-            net = {};
-            netstat = {};
-            nstat = {};
-            processes = {};
-            swap = {};
-            system = {};
-            systemd_units = {
-              unittype = "service";
-            };
-            temp = {};
-            wireguard = {};
-            # http_response = { urls = [ "http://localhost/" ]; };
-            # ping = { urls = [ "9.9.9.9" ]; };
-          }
-          // optionalAttrs config.services.smartd.enable {
-            sensors = {};
-            smart = {
-              attributes = true;
-              path_nvme = config.security.elewrap.telegraf-nvme.path;
-              path_smartctl = config.security.elewrap.telegraf-smartctl.path;
-              use_sudo = false;
-            };
-          }
-          // optionalAttrs config.services.nginx.enable {
-            nginx.urls = ["http://localhost/nginx_status"];
-          }
-          // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
-            wireless = {};
-          };
-      };
-    };
-
-    services.nginx.virtualHosts = mkIf config.services.nginx.enable {
-      localhost.listenAddresses = ["127.0.0.1" "[::1]"];
-      localhost.locations."= /nginx_status".extraConfig = ''
-        allow 127.0.0.0/8;
-        allow ::1;
-        deny all;
-        stub_status;
-        access_log off;
-      '';
-    };
-
-    systemd.services.telegraf = {
-      path = [
-        # Make sensors refer to the correct wrapper
-        (mkIf config.services.smartd.enable
-          (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path))
       ];
-      serviceConfig = {
+
-        # For wireguard statistics
+      age.secrets.telegraf-influxdb-token = {
-        AmbientCapabilities = ["CAP_NET_ADMIN"];
+        generator = {
-        RestartSec = "600"; # Retry every 10 minutes
+          script = args: ''
+            echo -n "INFLUX_TOKEN="
+            ${nodes.${cfg.influxdb2.node}.config.services.influxdb2.provision.retrieveToken tokenDef args}
+          '';
+          tags = ["influxdb"];
+        };
+        mode = "440";
+        group = "telegraf";
+      };
+
+      security.elewrap.telegraf-sensors = mkIf cfg.scrapeSensors {
+        command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"];
+        targetUser = "root";
+        allowedUsers = ["telegraf"];
+      };
+
+      security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable {
+        command = ["${pkgs.nvme-cli}/bin/nvme"];
+        targetUser = "root";
+        allowedUsers = ["telegraf"];
+        passArguments = true;
+      };
+
+      security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable {
+        command = ["${pkgs.smartmontools}/bin/smartctl"];
+        targetUser = "root";
+        allowedUsers = ["telegraf"];
+        passArguments = true;
+      };
+
+      services.telegraf = {
+        enable = true;
+        environmentFiles = [config.age.secrets.telegraf-influxdb-token.path];
+        extraConfig = {
+          agent = {
+            interval = "10s";
+            round_interval = true; # Always collect on :00,:10,...
+            metric_batch_size = 5000;
+            metric_buffer_limit = 50000;
+            collection_jitter = "0s";
+            flush_interval = "20s";
+            flush_jitter = "5s";
+            precision = "1ms";
+            hostname = config.node.name;
+            omit_hostname = false;
+          };
+          outputs = {
+            influxdb_v2 = {
+              urls = ["https://${cfg.influxdb2.domain}"];
+              token = "$INFLUX_TOKEN";
+              inherit (cfg.influxdb2) organization bucket;
+            };
+          };
+          inputs =
+            {
+              conntrack = {};
+              cpu = {};
+              disk = {};
+              diskio = {};
+              internal = {};
+              interrupts = {};
+              kernel = {};
+              kernel_vmstat = {};
+              linux_sysctl_fs = {};
+              mem = {};
+              net = {};
+              netstat = {};
+              nstat = {};
+              processes = {};
+              swap = {};
+              system = {};
+              systemd_units = {
+                unittype = "service";
+              };
+              temp = {};
+              wireguard = {};
+              # http_response = { urls = [ "http://localhost/" ]; };
+              # ping = { urls = [ "9.9.9.9" ]; };
+            }
+            // optionalAttrs config.services.smartd.enable {
+              sensors = {};
+              smart = {
+                attributes = true;
+                path_nvme = config.security.elewrap.telegraf-nvme.path;
+                path_smartctl = config.security.elewrap.telegraf-smartctl.path;
+                use_sudo = false;
+              };
+            }
+            // optionalAttrs config.services.nginx.enable {
+              nginx.urls = ["http://localhost/nginx_status"];
+            }
+            // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
+              wireless = {};
+            };
+        };
+      };
+
+      services.nginx.virtualHosts = mkIf config.services.nginx.enable {
+        localhost.listenAddresses = ["127.0.0.1" "[::1]"];
+        localhost.locations."= /nginx_status".extraConfig = ''
+          allow 127.0.0.0/8;
+          allow ::1;
+          deny all;
+          stub_status;
+          access_log off;
+        '';
+      };
+
+      systemd.services.telegraf = {
+        path = [
+          # Make sensors refer to the correct wrapper
+          (mkIf cfg.scrapeSensors
+            (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path))
+        ];
+        serviceConfig = {
+          # For wireguard statistics
+          AmbientCapabilities = ["CAP_NET_ADMIN"];
+          RestartSec = "600"; # Retry every 10 minutes
+        };
       };
     };
-  };
 }