feat(monitoring): remove location, add nginx upstream monitoring option

hosts/sire/guests/ai.nix

@@ -51,10 +51,10 @@ in {
   };
 
   globals.services.open-webui.domain = openWebuiDomain;
-  globals.monitoring.http.ollama-webui = {
-    url = "https://${openWebuiDomain}";
-    location = "home";
-    network = "internet";
+  globals.monitoring.http.ollama = {
+    url = config.services.open-webui.environment.OLLAMA_BASE_URL;
+    expectedBodyRegex = "Ollama is running";
+    network = "local-${config.node.name}";
   };
 
   nodes.sentinel = {
@@ -65,6 +65,10 @@ in {
           zone open-webui 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "Open WebUI";
+        };
       };
       virtualHosts.${openWebuiDomain} = {
         forceSSL = true;

hosts/sire/guests/grafana.nix

@@ -80,7 +80,7 @@ in {
   globals.services.grafana.domain = grafanaDomain;
   globals.monitoring.http.grafana = {
     url = "https://${grafanaDomain}";
-    location = "home";
+    expectedBodyRegex = "Grafana";
     network = "internet";
   };
 
@@ -96,6 +96,10 @@ in {
           zone grafana 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "Grafana";
+        };
       };
       virtualHosts.${grafanaDomain} = {
         forceSSL = true;
@@ -116,6 +120,10 @@ in {
           zone grafana 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "Grafana";
+        };
       };
       virtualHosts.${grafanaDomain} = {
         forceSSL = true;

hosts/sire/guests/immich.nix

@@ -193,7 +193,7 @@ in {
   globals.services.immich.domain = immichDomain;
   globals.monitoring.http.immich = {
     url = "https://${immichDomain}";
-    location = "home";
+    expectedBodyRegex = "immutable.entry.app";
     network = "internet";
   };
 
@@ -205,6 +205,10 @@ in {
           zone immich 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "immutable.entry.app";
+        };
       };
       virtualHosts.${immichDomain} = {
         forceSSL = true;
@@ -228,6 +232,10 @@ in {
           zone immich 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "immutable.entry.app";
+        };
       };
       virtualHosts.${immichDomain} = {
         forceSSL = true;

hosts/sire/guests/influxdb.nix

@@ -1,6 +1,5 @@
 {
   config,
-  globals,
   lib,
   nodes,
   pkgs,
@@ -28,10 +27,6 @@ in {
   };
 
   meta.telegraf.secrets."@GITHUB_ACCESS_TOKEN@" = config.age.secrets.github-access-token.path;
-  meta.telegraf.globalMonitoring = {
-    enable = true;
-    availableNetworks = ["internet" "home-wan" "home-lan"];
-  };
   services.telegraf.extraConfig.outputs.influxdb_v2.urls = lib.mkForce ["http://localhost:${toString influxdbPort}"];
 
   services.telegraf.extraConfig.inputs = {
@@ -49,11 +44,6 @@ in {
   };
 
   globals.services.influxdb.domain = influxdbDomain;
-  globals.monitoring.http.influxdb = {
-    url = "https://${influxdbDomain}";
-    location = "home";
-    network = "internet";
-  };
 
   nodes.sentinel = {
     services.nginx = {
@@ -63,6 +53,10 @@ in {
           zone influxdb 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "InfluxDB";
+        };
       };
       virtualHosts.${influxdbDomain} = let
         accessRules = ''
@@ -97,6 +91,10 @@ in {
           zone influxdb 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "InfluxDB";
+        };
       };
       virtualHosts.${influxdbDomain} = let
         accessRules = ''

hosts/sire/guests/loki.nix

@@ -18,11 +18,6 @@ in {
   };
 
   globals.services.loki.domain = lokiDomain;
-  globals.monitoring.http.loki = {
-    url = "https://${lokiDomain}";
-    location = "home";
-    network = "internet";
-  };
 
   nodes.sentinel = {
     age.secrets.loki-basic-auth-hashes = {
@@ -38,6 +33,11 @@ in {
           zone loki 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          path = "/ready";
+          expectedBodyRegex = "^ready";
+        };
       };
       virtualHosts.${lokiDomain} = {
         forceSSL = true;
@@ -80,6 +80,11 @@ in {
           zone loki 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          path = "/ready";
+          expectedBodyRegex = "^ready";
+        };
       };
       virtualHosts.${lokiDomain} = {
         forceSSL = true;

hosts/sire/guests/minecraft.nix

@@ -364,7 +364,11 @@ in {
   globals.monitoring.tcp.minecraft = {
     host = minecraftDomain;
     port = 25565;
-    location = "home";
+    network = "internet";
+  };
+  globals.monitoring.http.minecraft-map = {
+    url = "https://${minecraftDomain}";
+    expectedBodyRegex = "Minecraft Dynamic Map";
     network = "internet";
   };
 
@@ -400,6 +404,10 @@ in {
           zone minecraft 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "Minecraft Dynamic Map";
+        };
       };
       virtualHosts.${minecraftDomain} = {
         forceSSL = true;

hosts/sire/guests/paperless.nix

@@ -27,7 +27,7 @@ in {
   globals.services.paperless.domain = paperlessDomain;
   globals.monitoring.http.paperless = {
     url = "https://${paperlessDomain}";
-    location = "home";
+    expectedBodyRegex = "Paperless-ngx";
     network = "internet";
   };
 
@@ -39,6 +39,10 @@ in {
           zone paperless 64k;
           keepalive 2;
         '';
+        # direct upstream monitoring doesn't work because
+        # paperless allowed hosts fails for ip-based queries.
+        # But that's fine, we just monitor it via the domain above anyway.
+        #monitoring.enable = true;
       };
       virtualHosts.${paperlessDomain} = {
         forceSSL = true;
@@ -63,6 +67,10 @@ in {
           zone paperless 64k;
           keepalive 2;
         '';
+        monitoring = {
+          enable = true;
+          expectedBodyRegex = "Paperless-ngx";
+        };
       };
       virtualHosts.${paperlessDomain} = {
         forceSSL = true;

hosts/sire/guests/samba.nix

@@ -136,7 +136,6 @@ in {
   globals.monitoring.tcp.samba = {
     host = globals.net.home-lan.hosts.sire-samba.ipv4;
     port = 445;
-    location = "home";
     network = "home-lan";
   };