feat: add telegraf module

hosts/common/core/default.nix

@@ -21,6 +21,7 @@
     ../../../modules/promtail.nix
     ../../../modules/proxied-domains.nix
     ../../../modules/repo.nix
+    ../../../modules/telegraf.nix
     ../../../modules/wireguard.nix
   ];
 

hosts/common/core/impermanence.nix

@@ -147,6 +147,14 @@
           group = "influxdb2";
           mode = "0700";
         }
+      ]
+      ++ lib.optionals config.services.telegraf.enable [
+        {
+          directory = "/var/lib/telegraf";
+          user = "telegraf";
+          group = "telegraf";
+          mode = "0700";
+        }
       ];
   };
 }

hosts/common/core/system.nix

@@ -436,5 +436,6 @@
     vaultwarden = uidGid 988;
     oauth2_proxy = uidGid 987;
     influxdb2 = uidGid 986;
+    telegraf = uidGid 985;
   };
 }

hosts/common/zfs.nix

@@ -1,6 +1,7 @@
 {
-  pkgs,
   config,
+  lib,
+  pkgs,
   ...
 }: {
   boot.supportedFilesystems = ["zfs"];
@@ -22,4 +23,8 @@
       interval = "weekly";
     };
   };
+
+  services.telegraf.extraConfig.inputs = lib.mkIf config.services.telegraf.enable {
+    zfs.poolMetrics = true;
+  };
 }

hosts/ward/default.nix

@@ -25,6 +25,16 @@
     proxy = "sentinel";
   };
 
+  extra.telegraf = {
+    enable = true;
+    proxy = "sentinel";
+    # TODO organization = "servers";
+    # TODO bucket = "telegraf";
+  };
+
+  # TODO track my github stats
+  # services.telegraf.extraConfig.inputs.github = {};
+
   extra.microvms.vms = let
     defaults = {
       system = "x86_64-linux";
@@ -53,9 +63,6 @@
   #minecraft = defineVm;
   #firefly
 
-  #prometheus
-  #influxdb
-
   #maddy = defineVm;
   #anonaddy = defineVm;
 

hosts/ward/microvms/grafana/default.nix

@@ -35,9 +35,8 @@ in {
     group = "grafana";
   };
 
-  age.secrets.grafana-influxdb-basic-auth-password = {
-    rekeyFile = ./secrets/grafana-influxdb-basic-auth-password.age;
-    generator = "alnum";
+  age.secrets.grafana-influxdb-token = {
+    rekeyFile = ./secrets/grafana-influxdb-token.age;
     mode = "440";
     group = "grafana";
   };
@@ -47,10 +46,6 @@ in {
       config.age.secrets.grafana-loki-basic-auth-password
     ];
 
-    age.secrets.influxdb-basic-auth-hashes.generator.dependencies = [
-      config.age.secrets.grafana-influxdb-basic-auth-password
-    ];
-
     proxiedDomains.grafana = grafanaDomain;
 
     services.nginx = {
@@ -126,10 +121,7 @@ in {
           access = "proxy";
           url = "https://${sentinelCfg.proxiedDomains.influxdb}";
           orgId = 1;
-          basicAuth = true;
-          basicAuthUser = "${nodeName}+grafana-influxdb-basic-auth-password";
-          secureJsonData.basicAuthPassword = "$__file{${config.age.secrets.grafana-influxdb-basic-auth-password.path}}";
-          #secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}";
+          secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}";
           jsonData.version = "Flux";
         }
         {

hosts/ward/microvms/grafana/secrets/grafana-influxdb-token.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> X25519 R7qu+/ASvapcZPEO2cXqEisYoiwSims+RSuCRCZsYgw
+lcYaTINCZY3g6DVCgejc3twlVdcMJEQE066ANZlYy8M
+-> piv-p256 xqSe8Q A1Wtem3odHpc6a3PSFShHkm2L8MeLbtv2ofmCfisFWXp
++OFZvkDksOUSmqOQVPLL5Ytk8eBkEmSYv2jc9jNOJWw
+-> @/*G-grease {vMz+#o gEzN;NY "+/{ +UP0
+SL0mOBt6VjU+vI+1fXKz52Ch0V7tgxX7k4emorUCC+GFOgo3BLrTu6R23Xhnoub3
+zw
+--- kKnHHEZuf1z2jYZpo5Zbvffh5AVG+VztRauG1mFkUYw
+
ßJŚkÍ%ńy Ŕc‹Ź›q>É÷Đ@Ě_lę=‡V�}a©"�_ŠIŇ„ő®x«É>ý.´îBînřóśBłCä)……­~•9ÝA†Z��‡çŰgd¶‰	UŔl~ Ş&ĘQ‡
—W6TžGţşv‰™Ć“/'vě©+

hosts/ward/microvms/influxdb/default.nix

@@ -27,15 +27,6 @@ in {
   nodes.sentinel = {
     proxiedDomains.influxdb = influxdbDomain;
 
-    age.secrets.influxdb-basic-auth-hashes = {
-      rekeyFile = ./secrets/influxdb-basic-auth-hashes.age;
-      # Copy only the script so the dependencies can be added by the nodes
-      # that define passwords (using distributed-config).
-      generator.script = config.age.generators.basic-auth.script;
-      mode = "440";
-      group = "nginx";
-    };
-
     services.nginx = {
       upstreams.influxdb = {
         servers."${config.services.influxdb2.settings.http-bind-address}" = {};
@@ -54,9 +45,8 @@ in {
           proxyWebsockets = true;
           extraConfig = ''
             satisfy any;
-            auth_basic "Authentication required";
-            auth_basic_user_file ${sentinelCfg.age.secrets.influxdb-basic-auth-hashes.path};
-            access_log off;
+            ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses};
+            deny all;
           '';
         };
       };

hosts/ward/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/zackbiene/default.nix

@@ -8,8 +8,9 @@
   imports = [
     ../common/core
     ../common/hardware/odroid-n2plus.nix
-    #../common/initrd-ssh.nix
+    ../common/initrd-ssh.nix
     ../common/zfs.nix
+    ../common/bios-boot.nix
 
     ./fs.nix
     ./net.nix
@@ -23,10 +24,7 @@
     ./zigbee2mqtt.nix
   ];
 
-  # TODO replace by bios-boot.nix
-  # and grub.devices = ... once disko is in use.
-  boot.loader.grub.enable = false;
-  boot.loader.generic-extlinux-compatible.enable = true;
+  # TODO boot.loader.grub.devices = ["/dev/disk/by-id/${config.repo.secrets.local.disk.main}"];
   console.earlySetup = true;
 
   # Fails if there are no SMART devices

hosts/zackbiene/hostapd.nix

@@ -10,6 +10,8 @@
   # Associates each known client to a unique password
   age.secrets.wifi-clients.rekeyFile = ./secrets/wifi-clients.age;
 
+  hardware.wirelessRegulatoryDatabase = true;
+
   services.hostapd = {
     enable = true;
     radios.wlan1 = {

modules/extra.nix

@@ -117,8 +117,20 @@ in {
         access_log syslog:server=unix:/dev/log,nohostname json_combined;
         ssl_ecdh_curve secp384r1;
       '';
+
+      virtualHosts.localhost = {
+        locations."= /nginx_status".extraConfig = ''
+          allow 127.0.0.0/8;
+          deny all;
+          stub_status;
+        '';
+      };
     };
 
     networking.firewall.allowedTCPPorts = optionals config.services.nginx.enable [80 443];
+
+    services.telegraf.extraConfig.inputs = mkIf config.services.nginx.enable {
+      nginx.urls = ["http://localhost/nginx_status"];
+    };
   };
 }

modules/telegraf.nix

@@ -0,0 +1,110 @@
+{
+  config,
+  lib,
+  nodeName,
+  nodePath,
+  nodes,
+  pkgs,
+  ...
+}: let
+  inherit
+    (lib)
+    mdDoc
+    mkEnableOption
+    mkIf
+    mkOption
+    optionalAttrs
+    types
+    ;
+
+  cfg = config.extra.telegraf;
+in {
+  options.extra.telegraf = {
+    enable = mkEnableOption (mdDoc "telegraf to push metrics to influx.");
+    proxy = mkOption {
+      type = types.str;
+      description = mdDoc "The node name of the proxy server which provides the influx api endpoint.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # Connect safely via wireguard to skip authentication
+    networking.hosts.${nodes.${cfg.proxy}.config.extra.wireguard."proxy-${cfg.proxy}".ipv4} = [nodes.${cfg.proxy}.config.proxiedDomains.influxdb];
+
+    age.secrets.telegraf-influxdb-token = {
+      rekeyFile = nodePath + "/secrets/telegraf-influxdb-token.age";
+      mode = "440";
+      group = "telegraf";
+    };
+
+    services.telegraf = {
+      enable = true;
+      environmentFiles = [config.age.secrets.telegraf-influxdb-token.path];
+      extraConfig = {
+        agent = {
+          interval = "10s";
+          round_interval = true; # Always collect on :00,:10,...
+          metric_batch_size = 5000;
+          metric_buffer_limit = 50000;
+          collection_jitter = "0s";
+          flush_interval = "20s";
+          flush_jitter = "5s";
+          precision = "1ms";
+          hostname = nodeName;
+          omit_hostname = false;
+        };
+        outputs = {
+          influxdb_v2 = {
+            urls = ["https://${nodes.${cfg.proxy}.config.proxiedDomains.influxdb}"];
+            token = "$INFLUX_TOKEN";
+            organization = "servers";
+            bucket = "telegraf";
+          };
+        };
+        inputs =
+          {
+            conntrack = {};
+            cpu = {};
+            disk = {};
+            diskio = {};
+            internal = {};
+            interrupts = {};
+            kernel = {};
+            kernel_vmstat = {};
+            linux_sysctl_fs = {};
+            mem = {};
+            net = {};
+            netstat = {};
+            nstat = {};
+            processes = {};
+            sensors = {};
+            swap = {};
+            system = {};
+            systemd_units = {unittype = "service";};
+            temp = {};
+            wireguard = {};
+            # http_response = { urls = [ "http://localhost/" ]; };
+            # ping = { urls = [ "9.9.9.9" ]; };
+          }
+          // optionalAttrs config.services.smartd.enable {
+            smart = {
+              path_nvme = "${pkgs.nvme-cli}/bin/nvme";
+              path_smartctl = "${pkgs.smartmontools}/bin/smartctl";
+              use_sudo = true;
+            };
+            # TODO } // optionalAttrs config.services.iwd.enable {
+            # TODO   wireless = { };
+          };
+      };
+    };
+
+    systemd.services.telegraf = {
+      path = [
+        "/run/wrappers"
+        pkgs.lm_sensors
+      ];
+      # For wireguard statistics
+      serviceConfig.AmbientCapabilities = ["CAP_NET_ADMIN"];
+    };
+  };
+}

modules/wireguard.nix

@@ -137,7 +137,6 @@
         assertion = stringLength wgCfg.linkName < 16;
         message = "${assertionPrefix}: The specified linkName '${wgCfg.linkName}' is too long (must be max 15 characters).";
       }
-      # TODO at least 3 network participants and (externalPeers != {} or someone has via set to us) -> ip forwarding
     ];
 
     networking.firewall.allowedUDPPorts =