feat: define global monitoring for each service and automatically configure telegraf based on it

globals.nix

@@ -0,0 +1,59 @@
+{config, ...}: let
+  inherit (config) globals;
+in {
+  globals = {
+    net = {
+      home-wan = {
+        cidrv4 = "192.168.178.0/24";
+        hosts.fritzbox.id = 1;
+        hosts.ward.id = 2;
+      };
+
+      home-lan = {
+        cidrv4 = "192.168.1.0/24";
+        cidrv6 = "fd10::/64";
+        hosts.ward.id = 1;
+        hosts.sire.id = 2;
+        hosts.ward-adguardhome.id = 3;
+        hosts.ward-web-proxy.id = 4;
+        hosts.sire-samba.id = 10;
+      };
+
+      proxy-home = {
+        cidrv4 = "10.44.0.0/24";
+        cidrv6 = "fd00:44::/120";
+      };
+    };
+
+    monitoring = {
+      dns.cloudflare = {
+        server = "1.1.1.1";
+        domain = ".";
+        location = "home";
+        network = "home-lan";
+      };
+
+      ping = {
+        cloudflare = {
+          hostv4 = "1.1.1.1";
+          hostv6 = "2606:4700:4700::1111";
+          location = "external";
+          network = "internet";
+        };
+
+        google = {
+          hostv4 = "8.8.8.8";
+          hostv6 = "2001:4860:4860::8888";
+          location = "external";
+          network = "internet";
+        };
+
+        fritz-box = {
+          hostv4 = globals.net.home-wan.hosts.fritzbox.ipv4;
+          location = "home";
+          network = "home-wan";
+        };
+      };
+    };
+  };
+}

hosts/envoy/net.nix

@@ -1,17 +1,28 @@
-{config, ...}: {
+{
+  config,
+  lib,
+  ...
+}: let
+  icfg = config.repo.secrets.local.networking.interfaces.wan;
+in {
   networking.hostId = config.repo.secrets.local.networking.hostId;
   networking.domain = config.repo.secrets.global.domains.mail.primary;
   networking.hosts."127.0.0.1" = ["mx1.${config.repo.secrets.global.domains.mail.primary}"];
 
+  globals.monitoring.ping.envoy = {
+    hostv4 = lib.net.cidr.ip icfg.hostCidrv4;
+    hostv6 = lib.net.cidr.ip icfg.hostCidrv6;
+    location = "external";
+    network = "internet";
+  };
+
   boot.initrd.systemd.network = {
     enable = true;
     networks = {inherit (config.systemd.network.networks) "10-wan";};
   };
 
   systemd.network.networks = {
-    "10-wan" = let
+    "10-wan" = {
-      icfg = config.repo.secrets.local.networking.interfaces.wan;
-    in {
       address = [
         icfg.hostCidrv4
         icfg.hostCidrv6

hosts/sentinel/net.nix

@@ -1,7 +1,20 @@
-{config, ...}: {
+{
+  config,
+  lib,
+  ...
+}: let
+  icfg = config.repo.secrets.local.networking.interfaces.wan;
+in {
   networking.hostId = config.repo.secrets.local.networking.hostId;
   networking.domain = config.repo.secrets.global.domains.me;
 
+  globals.monitoring.ping.sentinel = {
+    hostv4 = lib.net.cidr.ip icfg.hostCidrv4;
+    hostv6 = lib.net.cidr.ip icfg.hostCidrv6;
+    location = "external";
+    network = "internet";
+  };
+
   # Forwarding required for forgejo 9922->22
   boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
 
@@ -11,9 +24,7 @@
   };
 
   systemd.network.networks = {
-    "10-wan" = let
+    "10-wan" = {
-      icfg = config.repo.secrets.local.networking.interfaces.wan;
-    in {
       address = [
         icfg.hostCidrv4
         icfg.hostCidrv6

hosts/sire/guests/ai.nix

@@ -51,6 +51,12 @@ in {
   };
 
   globals.services.open-webui.domain = openWebuiDomain;
+  globals.monitoring.http.ollama-webui = {
+    url = "https://${openWebuiDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.open-webui = {

hosts/sire/guests/grafana.nix

@@ -78,6 +78,12 @@ in {
   };
 
   globals.services.grafana.domain = grafanaDomain;
+  globals.monitoring.http.grafana = {
+    url = "https://${grafanaDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     age.secrets.loki-basic-auth-hashes.generator.dependencies = [
       config.age.secrets.grafana-loki-basic-auth-password

hosts/sire/guests/immich.nix

@@ -191,6 +191,12 @@ in {
   };
 
   globals.services.immich.domain = immichDomain;
+  globals.monitoring.http.immich = {
+    url = "https://${immichDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.immich = {

hosts/sire/guests/influxdb.nix

@@ -28,58 +28,13 @@ in {
   };
 
   meta.telegraf.secrets."@GITHUB_ACCESS_TOKEN@" = config.age.secrets.github-access-token.path;
+  meta.telegraf.globalMonitoring = {
+    enable = true;
+    availableNetworks = ["internet" "home-wan" "home-lan"];
+  };
   services.telegraf.extraConfig.outputs.influxdb_v2.urls = lib.mkForce ["http://localhost:${toString influxdbPort}"];
 
-  globals.monitoring.ping.cloudflare-dns = {
-    host = "1.1.1.1";
-    location = "external";
-  };
-
-  globals.monitoring.ping.google-dns = {
-    host = "8.8.8.8";
-    location = "external";
-  };
-
   services.telegraf.extraConfig.inputs = {
-    ping = [
-      {
-        method = "native";
-        urls = [
-          globals.net.home-wan.hosts.fritzbox.ipv4
-          globals.net.home-lan.hosts.ward.ipv4
-        ];
-        tags.type = "internal";
-        fieldpass = [
-          "percent_packet_loss"
-          "average_response_ms"
-        ];
-      }
-      {
-        method = "native";
-        urls = [
-          "1.1.1.1"
-          "8.8.8.8"
-          config.repo.secrets.global.domains.me
-          config.repo.secrets.global.domains.personal
-        ];
-        tags.type = "external";
-        fieldpass = [
-          "percent_packet_loss"
-          "average_response_ms"
-        ];
-      }
-    ];
-
-    # FIXME: pls define this on the relevant hosts. Then we can ping it from multiple other hosts
-    #http_response = [
-    #  {
-    #    urls = [
-    #    ];
-    #    response_string_match = "Index of /";
-    #    response_status_code = 200;
-    #  }
-    #];
-
     github = {
       access_token = "@GITHUB_ACCESS_TOKEN@";
       repositories = [
@@ -94,6 +49,12 @@ in {
   };
 
   globals.services.influxdb.domain = influxdbDomain;
+  globals.monitoring.http.influxdb = {
+    url = "https://${influxdbDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.influxdb = {

hosts/sire/guests/loki.nix

@@ -18,6 +18,12 @@ in {
   };
 
   globals.services.loki.domain = lokiDomain;
+  globals.monitoring.http.loki = {
+    url = "https://${lokiDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     age.secrets.loki-basic-auth-hashes = {
       generator.script = "basic-auth";

hosts/sire/guests/minecraft.nix

@@ -1,3 +1,4 @@
+# FIXME: todo: host the proxy on sentinel so the IPs are not lost in natting
 {
   config,
   pkgs,
@@ -360,6 +361,13 @@ in {
   ];
 
   globals.services.minecraft.domain = minecraftDomain;
+  globals.monitoring.tcp.minecraft = {
+    host = minecraftDomain;
+    port = 25565;
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     # Rewrite destination addr with dnat on incoming connections
     # and masquerade responses to make them look like they originate from this host.

hosts/sire/guests/paperless.nix

@@ -25,6 +25,12 @@ in {
   };
 
   globals.services.paperless.domain = paperlessDomain;
+  globals.monitoring.http.paperless = {
+    url = "https://${paperlessDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.paperless = {

hosts/sire/guests/samba.nix

@@ -133,6 +133,13 @@ in {
     openFirewall = true;
   };
 
+  globals.monitoring.tcp.samba = {
+    host = globals.net.home-lan.hosts.sire-samba.ipv4;
+    port = 445;
+    location = "home";
+    network = "home-lan";
+  };
+
   services.samba = {
     enable = true;
     openFirewall = true;

hosts/sire/net.nix

@@ -1,10 +1,18 @@
 {
   config,
   globals,
+  lib,
   ...
 }: {
   networking.hostId = config.repo.secrets.local.networking.hostId;
 
+  globals.monitoring.ping.sire = {
+    hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv4;
+    hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv6;
+    location = "home";
+    network = "home-lan";
+  };
+
   boot.initrd.systemd.network = {
     enable = true;
     networks."10-lan" = {

hosts/ward/guests/adguardhome.nix

@@ -13,6 +13,18 @@ in {
   };
 
   globals.services.adguardhome.domain = adguardhomeDomain;
+  globals.monitoring.dns.adguardhome = {
+    server = globals.net.home-lan.hosts.ward-adguardhome.ipv4;
+    domain = ".";
+    location = "home";
+    network = "home-lan";
+  };
+  globals.monitoring.http.adguardhome = {
+    url = "https://${adguardhomeDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.adguardhome = {

hosts/ward/guests/forgejo.nix

@@ -23,6 +23,12 @@ in {
   };
 
   globals.services.forgejo.domain = forgejoDomain;
+  globals.monitoring.http.forgejo = {
+    url = "https://${forgejoDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     # Rewrite destination addr with dnat on incoming connections
     # and masquerade responses to make them look like they originate from this host.

hosts/ward/guests/kanidm.nix

@@ -40,6 +40,12 @@ in {
   age.secrets.kanidm-oauth2-web-sentinel = mkRandomSecret;
 
   globals.services.kanidm.domain = kanidmDomain;
+  globals.monitoring.http.kanidm = {
+    url = "https://${kanidmDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.kanidm = {

hosts/ward/guests/netbird.nix

@@ -78,6 +78,12 @@ in {
   };
 
   globals.services.netbird.domain = netbirdDomain;
+  globals.monitoring.http.netbird = {
+    url = "https://${netbirdDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.netbird-mgmt = {

hosts/ward/guests/radicale.nix

@@ -7,6 +7,12 @@ in {
   };
 
   globals.services.radicale.domain = radicaleDomain;
+  globals.monitoring.http.radicale = {
+    url = "https://${radicaleDomain}";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.radicale = {

hosts/ward/guests/vaultwarden.nix

@@ -26,6 +26,13 @@ in {
   ];
 
   globals.services.vaultwarden.domain = vaultwardenDomain;
+  globals.monitoring.http.vaultwarden = {
+    url = "https://${vaultwardenDomain}";
+    expectedBodyRegex = "Vaultwarden";
+    location = "home";
+    network = "internet";
+  };
+
   nodes.sentinel = {
     services.nginx = {
       upstreams.vaultwarden = {

hosts/ward/net.nix

@@ -1,31 +1,16 @@
 {
   config,
   globals,
+  lib,
   ...
 }: {
   networking.hostId = config.repo.secrets.local.networking.hostId;
 
-  globals.net = {
+  globals.monitoring.ping.ward = {
-    home-wan = {
+    hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv4;
-      cidrv4 = "192.168.178.0/24";
+    hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv6;
-      hosts.fritzbox.id = 1;
+    location = "home";
-      hosts.ward.id = 2;
+    network = "home-lan";
-    };
-
-    home-lan = {
-      cidrv4 = "192.168.1.0/24";
-      cidrv6 = "fd10::/64";
-      hosts.ward.id = 1;
-      hosts.sire.id = 2;
-      hosts.ward-adguardhome.id = 3;
-      hosts.ward-web-proxy.id = 4;
-      hosts.sire-samba.id = 10;
-    };
-
-    proxy-home = {
-      cidrv4 = "10.44.0.0/24";
-      cidrv6 = "fd00:44::/120";
-    };
   };
 
   boot.initrd.systemd.network = {

hosts/zackbiene/net.nix

@@ -9,6 +9,13 @@
 in {
   networking.hostId = config.repo.secrets.local.networking.hostId;
 
+  globals.monitoring.ping.zackbiene = {
+    hostv4 = "zackbiene.local";
+    hostv6 = "zackbiene.local";
+    location = "home";
+    network = "home-lan";
+  };
+
   wireguard.proxy-home.client.via = "ward";
 
   boot.initrd.systemd.network = {

modules/globals.nix

@@ -98,15 +98,118 @@ in {
             ping = mkOption {
               type = types.attrsOf (types.submodule {
                 options = {
-                  host = mkOption {
+                  hostv4 = mkOption {
-                    type = types.str;
+                    type = types.nullOr types.str;
-                    description = "The IP/hostname to ping.";
+                    description = "The IP/hostname to ping via ipv4.";
+                    default = null;
+                  };
+
+                  hostv6 = mkOption {
+                    type = types.nullOr types.str;
+                    description = "The IP/hostname to ping via ipv6.";
+                    default = null;
                   };
 
                   location = mkOption {
                     type = types.str;
                     description = "A location tag added to this metric.";
                   };
+
+                  network = mkOption {
+                    type = types.str;
+                    description = "The network to which this endpoint is associated.";
+                  };
+                };
+              });
+            };
+
+            http = mkOption {
+              type = types.attrsOf (types.submodule {
+                options = {
+                  url = mkOption {
+                    type = types.str;
+                    description = "The url to connect to.";
+                  };
+
+                  location = mkOption {
+                    type = types.str;
+                    description = "A location tag added to this metric.";
+                  };
+
+                  network = mkOption {
+                    type = types.str;
+                    description = "The network to which this endpoint is associated.";
+                  };
+
+                  expectedStatus = mkOption {
+                    type = types.int;
+                    default = 200;
+                    description = "The HTTP status code to expect.";
+                  };
+
+                  expectedBodyRegex = mkOption {
+                    type = types.nullOr types.str;
+                    description = "A regex pattern to expect in the body.";
+                    default = null;
+                  };
+                };
+              });
+            };
+
+            dns = mkOption {
+              type = types.attrsOf (types.submodule {
+                options = {
+                  server = mkOption {
+                    type = types.str;
+                    description = "The DNS server to query.";
+                  };
+
+                  domain = mkOption {
+                    type = types.str;
+                    description = "The domain to query.";
+                  };
+
+                  record-type = mkOption {
+                    type = types.str;
+                    description = "The record type to query.";
+                    default = "A";
+                  };
+
+                  location = mkOption {
+                    type = types.str;
+                    description = "A location tag added to this metric.";
+                  };
+
+                  network = mkOption {
+                    type = types.str;
+                    description = "The network to which this endpoint is associated.";
+                  };
+                };
+              });
+            };
+
+            tcp = mkOption {
+              type = types.attrsOf (types.submodule {
+                options = {
+                  host = mkOption {
+                    type = types.str;
+                    description = "The IP/hostname to connect to.";
+                  };
+
+                  port = mkOption {
+                    type = types.port;
+                    description = "The port to connect to.";
+                  };
+
+                  location = mkOption {
+                    type = types.str;
+                    description = "A location tag added to this metric.";
+                  };
+
+                  network = mkOption {
+                    type = types.str;
+                    description = "The network to which this endpoint is associated.";
+                  };
                 };
               });
             };

modules/telegraf.nix

@@ -1,5 +1,6 @@
 {
   config,
+  globals,
   lib,
   minimal,
   nodes,
@@ -8,11 +9,18 @@
 }: let
   inherit
     (lib)
+    concatLists
+    elem
+    flip
+    forEach
+    mapAttrsToList
     mkAfter
     mkEnableOption
     mkIf
     mkOption
+    optional
     optionalAttrs
+    optionals
     types
     ;
 
@@ -36,6 +44,18 @@ in {
       description = "Additional secrets to replace in pre-start. The attr name will be searched and replaced in the config with the value read from the given file.";
     };
 
+    globalMonitoring = {
+      enable = mkEnableOption "monitor the global infrastructure from this node.";
+      availableNetworks = mkOption {
+        type = types.listOf types.str;
+        example = ["internet"];
+        description = ''
+          The networks that can be reached from this node.
+          Only global entries with a matching network will be monitored from here.
+        '';
+      };
+    };
+
     influxdb2 = {
       domain = mkOption {
         type = types.str;
@@ -165,8 +185,6 @@ in {
             };
             temp = {};
             wireguard = {};
-            # http_response = { urls = [ "http://localhost/" ]; };
-            # ping = { urls = [ "9.9.9.9" ]; };
           }
           // optionalAttrs config.services.smartd.enable {
             sensors = {};
@@ -182,6 +200,74 @@ in {
           }
           // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
             wireless = {};
+          }
+          // optionalAttrs cfg.globalMonitoring.enable {
+            ping = concatLists (flip mapAttrsToList globals.monitoring.ping (
+              name: pingCfg:
+                optionals (elem pingCfg.network cfg.globalMonitoring.availableNetworks) (
+                  concatLists (forEach ["hostv4" "hostv6"] (
+                    attr:
+                      optional (pingCfg.${attr} != null) {
+                        method = "native";
+                        urls = [pingCfg.${attr}];
+                        ipv4 = attr == "hostv4";
+                        ipv6 = attr == "hostv6";
+                        tags = {
+                          inherit name;
+                          inherit (pingCfg) location network;
+                          ip_version =
+                            if attr == "hostv4"
+                            then "v4"
+                            else "v6";
+                        };
+                        fieldpass = [
+                          "percent_packet_loss"
+                          "average_response_ms"
+                        ];
+                      }
+                  ))
+                )
+            ));
+
+            http_response = concatLists (flip mapAttrsToList globals.monitoring.http (
+              name: httpCfg:
+                optional (elem httpCfg.network cfg.globalMonitoring.availableNetworks) {
+                  urls = [httpCfg.url];
+                  method = "GET";
+                  response_status_code = httpCfg.expectedStatus;
+                  response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex;
+                  tags = {
+                    inherit name;
+                    inherit (httpCfg) location network;
+                  };
+                }
+            ));
+
+            dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns (
+              name: dnsCfg:
+                optional (elem dnsCfg.network cfg.globalMonitoring.availableNetworks) {
+                  servers = [dnsCfg.server];
+                  domains = [dnsCfg.domain];
+                  record_type = dnsCfg.record-type;
+                  tags = {
+                    inherit name;
+                    inherit (dnsCfg) location network;
+                  };
+                }
+            ));
+
+            net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp (
+              name: tcpCfg:
+                optional (elem tcpCfg.network cfg.globalMonitoring.availableNetworks) {
+                  address = "${tcpCfg.host}:${toString tcpCfg.port}";
+                  protocol = "tcp";
+                  tags = {
+                    inherit name;
+                    inherit (tcpCfg) location network;
+                  };
+                  fieldexclude = ["result_type" "string_found"];
+                }
+            ));
           };
       };
     };

nix/globals.nix

@@ -12,6 +12,7 @@
         };
         modules = [
           ../modules/globals.nix
+          ../globals.nix
           ({lib, ...}: {
             globals = lib.mkMerge (
               lib.concatLists (lib.flip lib.mapAttrsToList config.nodes (