feat: enable telegraf on all server nodes; add 10 minute autorestart

2025-10-11 07:10:39 +02:00 · 2023-06-25 02:34:05 +02:00 · 2023-06-25 02:34:05 +02:00 · f606e6e554
commit f606e6e554
parent b8f647fb4a
25 changed files with 228 additions and 41 deletions
--- a/hosts/common/core/default.nix
+++ b/hosts/common/core/default.nix
@ -19,7 +19,7 @@
    ../../../modules/microvms.nix
    ../../../modules/oauth2-proxy.nix
    ../../../modules/promtail.nix
-    ../../../modules/proxied-domains.nix
+    ../../../modules/provided-domains.nix
    ../../../modules/repo.nix
    ../../../modules/telegraf.nix
    ../../../modules/wireguard.nix
--- a/hosts/sentinel/default.nix
+++ b/hosts/sentinel/default.nix
@ -24,4 +24,13 @@
    enable = true;
    proxy = "sentinel";
  };
+
+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${config.extra.wireguard.proxy-sentinel.ipv4} = [config.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = config.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
 }
--- a/hosts/sentinel/net.nix
+++ b/hosts/sentinel/net.nix
@ -40,6 +40,15 @@
  networking.nftables.firewall = {
    zones = lib.mkForce {
      untrusted.interfaces = ["wan"];
+      proxy-sentinel.interfaces = ["proxy-sentinel"];
+    };
+    rules = lib.mkForce {
+      # Allow accessing nginx through the proxy
+      proxy-sentinel-to-local = {
+        from = ["proxy-sentinel"];
+        to = ["local"];
+        allowedTCPPorts = [80 443];
+      };
    };
  };

--- a/hosts/sentinel/oauth2.nix
+++ b/hosts/sentinel/oauth2.nix
@ -22,15 +22,15 @@
  in {
    provider = "oidc";
    scope = "openid email";
-    loginURL = "https://${config.proxiedDomains.kanidm}/ui/oauth2";
-    redeemURL = "https://${config.proxiedDomains.kanidm}/oauth2/token";
-    validateURL = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
+    loginURL = "https://${config.providedDomains.kanidm}/ui/oauth2";
+    redeemURL = "https://${config.providedDomains.kanidm}/oauth2/token";
+    validateURL = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
    clientID = clientId;
    keyFile = config.age.secrets.oauth2-proxy-secret.path;
    email.domains = ["*"];

    extraConfig = {
-      oidc-issuer-url = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}";
+      oidc-issuer-url = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}";
      provider-display-name = "Kanidm";
      #skip-provider-button = true;
    };
--- a/hosts/sentinel/secrets/telegraf-influxdb-token.age
+++ b/hosts/sentinel/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/default.nix
+++ b/hosts/ward/default.nix
@ -1,6 +1,7 @@
 {
  config,
  nixos-hardware,
+  nodes,
  ...
 }: {
  imports = [
@ -25,11 +26,13 @@
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${nodes.sentinel.config.extra.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.providedDomains.influxdb];
  extra.telegraf = {
    enable = true;
-    proxy = "sentinel";
-    # TODO organization = "servers";
-    # TODO bucket = "telegraf";
+    influxdb2.url = nodes.sentinel.config.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
  };

  # TODO track my github stats
--- a/hosts/ward/microvms/adguardhome/default.nix
+++ b/hosts/ward/microvms/adguardhome/default.nix
@ -17,12 +17,21 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  networking.nftables.firewall.rules = lib.mkForce {
    sentinel-to-local.allowedTCPPorts = [config.services.adguardhome.settings.bind_port];
  };

  nodes.sentinel = {
-    proxiedDomains.adguard = adguardhomeDomain;
+    providedDomains.adguard = adguardhomeDomain;

    services.nginx = {
      upstreams.adguardhome = {
--- a/hosts/ward/microvms/adguardhome/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/adguardhome/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/microvms/grafana/default.nix
+++ b/hosts/ward/microvms/grafana/default.nix
@ -18,6 +18,15 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  networking.nftables.firewall.rules = lib.mkForce {
    sentinel-to-local.allowedTCPPorts = [config.services.grafana.settings.server.http_port];
  };
@ -46,7 +55,7 @@ in {
      config.age.secrets.grafana-loki-basic-auth-password
    ];

-    proxiedDomains.grafana = grafanaDomain;
+    providedDomains.grafana = grafanaDomain;

    services.nginx = {
      upstreams.grafana = {
@ -102,9 +111,9 @@ in {
        client_secret = "aZKNCM6KpjBy4RqwKJXMLXzyx9rKH6MZTFk4wYrKWuBqLj6t"; # TODO temporary test not a real secret
        scopes = "openid email profile";
        login_attribute_path = "prefered_username";
-        auth_url = "https://${sentinelCfg.proxiedDomains.kanidm}/ui/oauth2";
-        token_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/token";
-        api_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/openid/grafana/userinfo";
+        auth_url = "https://${sentinelCfg.providedDomains.kanidm}/ui/oauth2";
+        token_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/token";
+        api_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/openid/grafana/userinfo";
        use_pkce = true;
        # Allow mapping oauth2 roles to server admin
        allow_assign_grafana_admin = true;
@ -116,19 +125,22 @@ in {
      enable = true;
      datasources.settings.datasources = [
        {
-          name = "InfluxDB";
+          name = "InfluxDB (servers)";
          type = "influxdb";
          access = "proxy";
-          url = "https://${sentinelCfg.proxiedDomains.influxdb}";
+          url = "https://${sentinelCfg.providedDomains.influxdb}";
          orgId = 1;
          secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}";
          jsonData.version = "Flux";
+          jsonData.organization = "servers";
+          jsonData.defaultBucket = "telegraf";
        }
+        # TODO duplicate above influxdb source (with scoped read tokens??) for each organization
        {
          name = "Loki";
          type = "loki";
          access = "proxy";
-          url = "https://${sentinelCfg.proxiedDomains.loki}";
+          url = "https://${sentinelCfg.providedDomains.loki}";
          orgId = 1;
          basicAuth = true;
          basicAuthUser = "${nodeName}+grafana-loki-basic-auth-password";
--- a/hosts/ward/microvms/grafana/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/grafana/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/microvms/influxdb/default.nix
+++ b/hosts/ward/microvms/influxdb/default.nix
@ -20,12 +20,21 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  networking.nftables.firewall.rules = lib.mkForce {
    sentinel-to-local.allowedTCPPorts = [influxdbPort];
  };

  nodes.sentinel = {
-    proxiedDomains.influxdb = influxdbDomain;
+    providedDomains.influxdb = influxdbDomain;

    services.nginx = {
      upstreams.influxdb = {
@ -45,7 +54,7 @@ in {
          proxyWebsockets = true;
          extraConfig = ''
            satisfy any;
-            ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses};
+            ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses}
            deny all;
          '';
        };
--- a/hosts/ward/microvms/influxdb/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/influxdb/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/microvms/kanidm/default.nix
+++ b/hosts/ward/microvms/kanidm/default.nix
@ -19,6 +19,15 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  networking.nftables.firewall.rules = lib.mkForce {
    sentinel-to-local.allowedTCPPorts = [kanidmPort];
  };
@ -36,7 +45,7 @@ in {
  };

  nodes.sentinel = {
-    proxiedDomains.kanidm = kanidmDomain;
+    providedDomains.kanidm = kanidmDomain;

    services.nginx = {
      upstreams.kanidm = {
--- a/hosts/ward/microvms/kanidm/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/kanidm/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/microvms/loki/default.nix
+++ b/hosts/ward/microvms/loki/default.nix
@ -17,12 +17,21 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  networking.nftables.firewall.rules = lib.mkForce {
    sentinel-to-local.allowedTCPPorts = [config.services.loki.configuration.server.http_listen_port];
  };

  nodes.sentinel = {
-    proxiedDomains.loki = lokiDomain;
+    providedDomains.loki = lokiDomain;

    age.secrets.loki-basic-auth-hashes = {
      rekeyFile = ./secrets/loki-basic-auth-hashes.age;
--- a/hosts/ward/microvms/loki/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/loki/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/microvms/vaultwarden/default.nix
+++ b/hosts/ward/microvms/vaultwarden/default.nix
@ -17,6 +17,15 @@ in {
    proxy = "sentinel";
  };

+  # Connect safely via wireguard to skip authentication
+  networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
+  extra.telegraf = {
+    enable = true;
+    influxdb2.url = sentinelCfg.providedDomains.influxdb;
+    influxdb2.organization = "servers";
+    influxdb2.bucket = "telegraf";
+  };
+
  age.secrets.vaultwarden-env = {
    rekeyFile = ./secrets/vaultwarden-env.age;
    mode = "440";
@ -31,7 +40,7 @@ in {
  };

  nodes.sentinel = {
-    proxiedDomains.vaultwarden = vaultwardenDomain;
+    providedDomains.vaultwarden = vaultwardenDomain;

    upstreams.vaultwarden = {
      servers."${config.services.vaultwarden.config.rocketAddress}:${toString config.services.vaultwarden.config.rocketPort}" = {};
--- a/hosts/ward/microvms/vaultwarden/secrets/telegraf-influxdb-token.age
+++ b/hosts/ward/microvms/vaultwarden/secrets/telegraf-influxdb-token.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
+KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
+-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
+SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
+-> e\9`z-grease
+PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
+RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
+6g
+--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
+¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}ÄÜ¶§W?ub
+)‘¯/û,{÷&ƒFÿ-ŒØ5£ß/u.�p¬	\%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
+§Óˆ[Áþ°t‡__4y× ±q�¬^/F×©*
--- a/hosts/ward/net.nix
+++ b/hosts/ward/net.nix
@ -177,4 +177,7 @@ in {
    macvtapInterface = "lan";
    wireguard.openFirewallRules = ["lan-to-local"];
  };
+
+  # Allow accessing influx
+  extra.wireguard.proxy-sentinel.client.via = "sentinel";
 }