feat: enable telegraf on all server nodes; add 10 minute autorestart

hosts/common/core/default.nix

@@ -19,7 +19,7 @@
     ../../../modules/microvms.nix
     ../../../modules/oauth2-proxy.nix
     ../../../modules/promtail.nix
-    ../../../modules/proxied-domains.nix
+    ../../../modules/provided-domains.nix
     ../../../modules/repo.nix
     ../../../modules/telegraf.nix
     ../../../modules/wireguard.nix

hosts/sentinel/default.nix

@@ -24,4 +24,13 @@
     enable = true;
     proxy = "sentinel";
   };
+
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${config.extra.wireguard.proxy-sentinel.ipv4} = [config.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = config.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
 }

hosts/sentinel/net.nix

@@ -40,6 +40,15 @@
   networking.nftables.firewall = {
     zones = lib.mkForce {
       untrusted.interfaces = ["wan"];
+      proxy-sentinel.interfaces = ["proxy-sentinel"];
+    };
+    rules = lib.mkForce {
+      # Allow accessing nginx through the proxy
+      proxy-sentinel-to-local = {
+        from = ["proxy-sentinel"];
+        to = ["local"];
+        allowedTCPPorts = [80 443];
+      };
     };
   };
 

hosts/sentinel/oauth2.nix

@@ -22,15 +22,15 @@
   in {
     provider = "oidc";
     scope = "openid email";
-    loginURL = "https://${config.proxiedDomains.kanidm}/ui/oauth2";
+    loginURL = "https://${config.providedDomains.kanidm}/ui/oauth2";
-    redeemURL = "https://${config.proxiedDomains.kanidm}/oauth2/token";
+    redeemURL = "https://${config.providedDomains.kanidm}/oauth2/token";
-    validateURL = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
+    validateURL = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
     clientID = clientId;
     keyFile = config.age.secrets.oauth2-proxy-secret.path;
     email.domains = ["*"];
 
     extraConfig = {
-      oidc-issuer-url = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}";
+      oidc-issuer-url = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}";
       provider-display-name = "Kanidm";
       #skip-provider-button = true;
     };

hosts/sentinel/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/default.nix

@@ -1,6 +1,7 @@
 {
   config,
   nixos-hardware,
+  nodes,
   ...
 }: {
   imports = [
@@ -25,11 +26,13 @@
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${nodes.sentinel.config.extra.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.providedDomains.influxdb];
   extra.telegraf = {
     enable = true;
-    proxy = "sentinel";
+    influxdb2.url = nodes.sentinel.config.providedDomains.influxdb;
-    # TODO organization = "servers";
+    influxdb2.organization = "servers";
-    # TODO bucket = "telegraf";
+    influxdb2.bucket = "telegraf";
   };
 
   # TODO track my github stats

hosts/ward/microvms/adguardhome/default.nix

@@ -17,12 +17,21 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   networking.nftables.firewall.rules = lib.mkForce {
     sentinel-to-local.allowedTCPPorts = [config.services.adguardhome.settings.bind_port];
   };
 
   nodes.sentinel = {
-    proxiedDomains.adguard = adguardhomeDomain;
+    providedDomains.adguard = adguardhomeDomain;
 
     services.nginx = {
       upstreams.adguardhome = {

hosts/ward/microvms/adguardhome/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/microvms/grafana/default.nix

@@ -18,6 +18,15 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   networking.nftables.firewall.rules = lib.mkForce {
     sentinel-to-local.allowedTCPPorts = [config.services.grafana.settings.server.http_port];
   };
@@ -46,7 +55,7 @@ in {
       config.age.secrets.grafana-loki-basic-auth-password
     ];
 
-    proxiedDomains.grafana = grafanaDomain;
+    providedDomains.grafana = grafanaDomain;
 
     services.nginx = {
       upstreams.grafana = {
@@ -102,9 +111,9 @@ in {
         client_secret = "aZKNCM6KpjBy4RqwKJXMLXzyx9rKH6MZTFk4wYrKWuBqLj6t"; # TODO temporary test not a real secret
         scopes = "openid email profile";
         login_attribute_path = "prefered_username";
-        auth_url = "https://${sentinelCfg.proxiedDomains.kanidm}/ui/oauth2";
+        auth_url = "https://${sentinelCfg.providedDomains.kanidm}/ui/oauth2";
-        token_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/token";
+        token_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/token";
-        api_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/openid/grafana/userinfo";
+        api_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/openid/grafana/userinfo";
         use_pkce = true;
         # Allow mapping oauth2 roles to server admin
         allow_assign_grafana_admin = true;
@@ -116,19 +125,22 @@ in {
       enable = true;
       datasources.settings.datasources = [
         {
-          name = "InfluxDB";
+          name = "InfluxDB (servers)";
           type = "influxdb";
           access = "proxy";
-          url = "https://${sentinelCfg.proxiedDomains.influxdb}";
+          url = "https://${sentinelCfg.providedDomains.influxdb}";
           orgId = 1;
           secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}";
           jsonData.version = "Flux";
+          jsonData.organization = "servers";
+          jsonData.defaultBucket = "telegraf";
         }
+        # TODO duplicate above influxdb source (with scoped read tokens??) for each organization
         {
           name = "Loki";
           type = "loki";
           access = "proxy";
-          url = "https://${sentinelCfg.proxiedDomains.loki}";
+          url = "https://${sentinelCfg.providedDomains.loki}";
           orgId = 1;
           basicAuth = true;
           basicAuthUser = "${nodeName}+grafana-loki-basic-auth-password";

hosts/ward/microvms/grafana/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/microvms/influxdb/default.nix

@@ -20,12 +20,21 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   networking.nftables.firewall.rules = lib.mkForce {
     sentinel-to-local.allowedTCPPorts = [influxdbPort];
   };
 
   nodes.sentinel = {
-    proxiedDomains.influxdb = influxdbDomain;
+    providedDomains.influxdb = influxdbDomain;
 
     services.nginx = {
       upstreams.influxdb = {
@@ -45,7 +54,7 @@ in {
           proxyWebsockets = true;
           extraConfig = ''
             satisfy any;
-            ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses};
+            ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses}
             deny all;
           '';
         };

hosts/ward/microvms/influxdb/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/microvms/kanidm/default.nix

@@ -19,6 +19,15 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   networking.nftables.firewall.rules = lib.mkForce {
     sentinel-to-local.allowedTCPPorts = [kanidmPort];
   };
@@ -36,7 +45,7 @@ in {
   };
 
   nodes.sentinel = {
-    proxiedDomains.kanidm = kanidmDomain;
+    providedDomains.kanidm = kanidmDomain;
 
     services.nginx = {
       upstreams.kanidm = {

hosts/ward/microvms/kanidm/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/microvms/loki/default.nix

@@ -17,12 +17,21 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   networking.nftables.firewall.rules = lib.mkForce {
     sentinel-to-local.allowedTCPPorts = [config.services.loki.configuration.server.http_listen_port];
   };
 
   nodes.sentinel = {
-    proxiedDomains.loki = lokiDomain;
+    providedDomains.loki = lokiDomain;
 
     age.secrets.loki-basic-auth-hashes = {
       rekeyFile = ./secrets/loki-basic-auth-hashes.age;

hosts/ward/microvms/loki/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/microvms/vaultwarden/default.nix

@@ -17,6 +17,15 @@ in {
     proxy = "sentinel";
   };
 
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
   age.secrets.vaultwarden-env = {
     rekeyFile = ./secrets/vaultwarden-env.age;
     mode = "440";
@@ -31,7 +40,7 @@ in {
   };
 
   nodes.sentinel = {
-    proxiedDomains.vaultwarden = vaultwardenDomain;
+    providedDomains.vaultwarden = vaultwardenDomain;
 
     upstreams.vaultwarden = {
       servers."${config.services.vaultwarden.config.rocketAddress}:${toString config.services.vaultwarden.config.rocketPort}" = {};

hosts/ward/microvms/vaultwarden/secrets/telegraf-influxdb-token.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

hosts/ward/net.nix

@@ -177,4 +177,7 @@ in {
     macvtapInterface = "lan";
     wireguard.openFirewallRules = ["lan-to-local"];
   };
+
+  # Allow accessing influx
+  extra.wireguard.proxy-sentinel.client.via = "sentinel";
 }

modules/distributed-config.nix

@@ -37,7 +37,7 @@ in {
     allNodes = attrNames colmenaNodes;
     isColmenaNode = elem nodeName allNodes;
     foreignConfigs = concatMap (n: colmenaNodes.${n}.config.nodes.${nodeName} or []) allNodes;
-    toplevelAttrs = ["age" "proxiedDomains" "networking" "systemd" "services"];
+    toplevelAttrs = ["age" "providedDomains" "networking" "systemd" "services"];
   in
     optionalAttrs isColmenaNode (mergeToplevelConfigs toplevelAttrs (
       foreignConfigs

modules/oauth2-proxy.nix

@@ -125,6 +125,7 @@ in {
       RuntimeDirectory = "oauth2_proxy";
       RuntimeDirectoryMode = "0750";
       UMask = "007"; # TODO remove once https://github.com/oauth2-proxy/oauth2-proxy/issues/2141 is fixed
+      RestartSec = "600"; # Retry every 10 minutes
     };
 
     users.groups.oauth2_proxy.members = ["nginx"];

modules/promtail.nix

@@ -50,7 +50,7 @@ in {
           {
             basic_auth.username = "${nodeName}+promtail-loki-basic-auth-password";
             basic_auth.password_file = config.age.secrets.promtail-loki-basic-auth-password.path;
-            url = "https://${nodes.${cfg.proxy}.config.proxiedDomains.loki}/loki/api/v1/push";
+            url = "https://${nodes.${cfg.proxy}.config.providedDomains.loki}/loki/api/v1/push";
           }
         ];
 
@@ -147,5 +147,7 @@ in {
         ];
       };
     };
+
+    systemd.services.promtail.serviceConfig.RestartSec = "600"; # Retry every 10 minutes
   };
 }

modules/provided-domains.nix

@@ -0,0 +1,7 @@
+{lib, ...}: {
+  options.providedDomains = lib.mkOption {
+    type = lib.types.attrsOf lib.types.str;
+    default = {};
+    description = "Registry of domains that this host 'provides' (that refer to this host with some functionality). For easy cross-node referencing.";
+  };
+}

modules/proxied-domains.nix

@@ -1,7 +0,0 @@
-{lib, ...}: {
-  options.proxiedDomains = lib.mkOption {
-    type = lib.types.attrsOf lib.types.str;
-    default = {};
-    description = "Registry of proxied domains for easy cross-node referencing.";
-  };
-}

modules/telegraf.nix

@@ -21,16 +21,26 @@
 in {
   options.extra.telegraf = {
     enable = mkEnableOption (mdDoc "telegraf to push metrics to influx.");
-    proxy = mkOption {
+    influxdb2 = {
+      url = mkOption {
         type = types.str;
-      description = mdDoc "The node name of the proxy server which provides the influx api endpoint.";
+        example = "https://influxdb.example.com";
+        description = mdDoc "The influxdb v2 database url to push to.";
+      };
+
+      organization = mkOption {
+        type = types.str;
+        description = mdDoc "The organization to push to.";
+      };
+
+      bucket = mkOption {
+        type = types.str;
+        description = mdDoc "The bucket to push to.";
+      };
     };
   };
 
   config = mkIf cfg.enable {
-    # Connect safely via wireguard to skip authentication
-    networking.hosts.${nodes.${cfg.proxy}.config.extra.wireguard."proxy-${cfg.proxy}".ipv4} = [nodes.${cfg.proxy}.config.proxiedDomains.influxdb];
-
     age.secrets.telegraf-influxdb-token = {
       rekeyFile = nodePath + "/secrets/telegraf-influxdb-token.age";
       mode = "440";
@@ -55,10 +65,9 @@ in {
         };
         outputs = {
           influxdb_v2 = {
-            urls = ["https://${nodes.${cfg.proxy}.config.proxiedDomains.influxdb}"];
+            urls = [cfg.influxdb2.url];
             token = "$INFLUX_TOKEN";
-            organization = "servers";
+            inherit (cfg.influxdb2) organization bucket;
-            bucket = "telegraf";
           };
         };
         inputs =
@@ -103,8 +112,11 @@ in {
         "/run/wrappers"
         pkgs.lm_sensors
       ];
+      serviceConfig = {
         # For wireguard statistics
-      serviceConfig.AmbientCapabilities = ["CAP_NET_ADMIN"];
+        AmbientCapabilities = ["CAP_NET_ADMIN"];
+        RestartSec = "600"; # Retry every 10 minutes
+      };
     };
   };
 }