diff --git a/globals.nix b/globals.nix index dfd1849..cf8b6ef 100644 --- a/globals.nix +++ b/globals.nix @@ -28,31 +28,35 @@ in { }; monitoring = { - dns.cloudflare = { - server = "1.1.1.1"; - domain = "."; - location = "home"; - network = "home-lan"; + dns = { + cloudflare = { + server = "1.1.1.1"; + domain = "."; + network = "internet"; + }; + + google = { + server = "8.8.8.8"; + domain = "."; + network = "internet"; + }; }; ping = { cloudflare = { hostv4 = "1.1.1.1"; hostv6 = "2606:4700:4700::1111"; - location = "external"; network = "internet"; }; google = { hostv4 = "8.8.8.8"; hostv6 = "2001:4860:4860::8888"; - location = "external"; network = "internet"; }; fritz-box = { hostv4 = globals.net.home-wan.hosts.fritzbox.ipv4; - location = "home"; network = "home-wan"; }; }; diff --git a/hosts/envoy/net.nix b/hosts/envoy/net.nix index d10a1a2..0eb32bf 100644 --- a/hosts/envoy/net.nix +++ b/hosts/envoy/net.nix @@ -12,7 +12,6 @@ in { globals.monitoring.ping.envoy = { hostv4 = lib.net.cidr.ip icfg.hostCidrv4; hostv6 = lib.net.cidr.ip icfg.hostCidrv6; - location = "external"; network = "internet"; }; diff --git a/hosts/sentinel/default.nix b/hosts/sentinel/default.nix index 3020043..a56c2ee 100644 --- a/hosts/sentinel/default.nix +++ b/hosts/sentinel/default.nix @@ -55,5 +55,8 @@ bucket = "telegraf"; node = "sire-influxdb"; }; + + # This node shall monitor the infrastructure + availableMonitoringNetworks = ["internet"]; }; } diff --git a/hosts/sentinel/net.nix b/hosts/sentinel/net.nix index 519b0f4..3b5803f 100644 --- a/hosts/sentinel/net.nix +++ b/hosts/sentinel/net.nix @@ -11,7 +11,6 @@ in { globals.monitoring.ping.sentinel = { hostv4 = lib.net.cidr.ip icfg.hostCidrv4; hostv6 = lib.net.cidr.ip icfg.hostCidrv6; - location = "external"; network = "internet"; }; diff --git a/hosts/sire/guests/ai.nix b/hosts/sire/guests/ai.nix index 679da2f..5f9cf6d 100644 --- a/hosts/sire/guests/ai.nix +++ b/hosts/sire/guests/ai.nix @@ -51,10 +51,10 @@ in { }; globals.services.open-webui.domain = openWebuiDomain; - globals.monitoring.http.ollama-webui = { - url = "https://${openWebuiDomain}"; - location = "home"; - network = "internet"; + globals.monitoring.http.ollama = { + url = config.services.open-webui.environment.OLLAMA_BASE_URL; + expectedBodyRegex = "Ollama is running"; + network = "local-${config.node.name}"; }; nodes.sentinel = { @@ -65,6 +65,10 @@ in { zone open-webui 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Open WebUI"; + }; }; virtualHosts.${openWebuiDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/grafana.nix b/hosts/sire/guests/grafana.nix index 845faed..1d11e61 100644 --- a/hosts/sire/guests/grafana.nix +++ b/hosts/sire/guests/grafana.nix @@ -80,7 +80,7 @@ in { globals.services.grafana.domain = grafanaDomain; globals.monitoring.http.grafana = { url = "https://${grafanaDomain}"; - location = "home"; + expectedBodyRegex = "Grafana"; network = "internet"; }; @@ -96,6 +96,10 @@ in { zone grafana 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Grafana"; + }; }; virtualHosts.${grafanaDomain} = { forceSSL = true; @@ -116,6 +120,10 @@ in { zone grafana 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Grafana"; + }; }; virtualHosts.${grafanaDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/immich.nix b/hosts/sire/guests/immich.nix index 6ea9fa3..8b3fbad 100644 --- a/hosts/sire/guests/immich.nix +++ b/hosts/sire/guests/immich.nix @@ -193,7 +193,7 @@ in { globals.services.immich.domain = immichDomain; globals.monitoring.http.immich = { url = "https://${immichDomain}"; - location = "home"; + expectedBodyRegex = "immutable.entry.app"; network = "internet"; }; @@ -205,6 +205,10 @@ in { zone immich 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "immutable.entry.app"; + }; }; virtualHosts.${immichDomain} = { forceSSL = true; @@ -228,6 +232,10 @@ in { zone immich 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "immutable.entry.app"; + }; }; virtualHosts.${immichDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/influxdb.nix b/hosts/sire/guests/influxdb.nix index 7a3448c..edbcde5 100644 --- a/hosts/sire/guests/influxdb.nix +++ b/hosts/sire/guests/influxdb.nix @@ -1,6 +1,5 @@ { config, - globals, lib, nodes, pkgs, @@ -28,10 +27,6 @@ in { }; meta.telegraf.secrets."@GITHUB_ACCESS_TOKEN@" = config.age.secrets.github-access-token.path; - meta.telegraf.globalMonitoring = { - enable = true; - availableNetworks = ["internet" "home-wan" "home-lan"]; - }; services.telegraf.extraConfig.outputs.influxdb_v2.urls = lib.mkForce ["http://localhost:${toString influxdbPort}"]; services.telegraf.extraConfig.inputs = { @@ -49,11 +44,6 @@ in { }; globals.services.influxdb.domain = influxdbDomain; - globals.monitoring.http.influxdb = { - url = "https://${influxdbDomain}"; - location = "home"; - network = "internet"; - }; nodes.sentinel = { services.nginx = { @@ -63,6 +53,10 @@ in { zone influxdb 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "InfluxDB"; + }; }; virtualHosts.${influxdbDomain} = let accessRules = '' @@ -97,6 +91,10 @@ in { zone influxdb 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "InfluxDB"; + }; }; virtualHosts.${influxdbDomain} = let accessRules = '' diff --git a/hosts/sire/guests/loki.nix b/hosts/sire/guests/loki.nix index ec43a38..42de6cd 100644 --- a/hosts/sire/guests/loki.nix +++ b/hosts/sire/guests/loki.nix @@ -18,11 +18,6 @@ in { }; globals.services.loki.domain = lokiDomain; - globals.monitoring.http.loki = { - url = "https://${lokiDomain}"; - location = "home"; - network = "internet"; - }; nodes.sentinel = { age.secrets.loki-basic-auth-hashes = { @@ -38,6 +33,11 @@ in { zone loki 64k; keepalive 2; ''; + monitoring = { + enable = true; + path = "/ready"; + expectedBodyRegex = "^ready"; + }; }; virtualHosts.${lokiDomain} = { forceSSL = true; @@ -80,6 +80,11 @@ in { zone loki 64k; keepalive 2; ''; + monitoring = { + enable = true; + path = "/ready"; + expectedBodyRegex = "^ready"; + }; }; virtualHosts.${lokiDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/minecraft.nix b/hosts/sire/guests/minecraft.nix index 7f4ce2a..ead8bcd 100644 --- a/hosts/sire/guests/minecraft.nix +++ b/hosts/sire/guests/minecraft.nix @@ -364,7 +364,11 @@ in { globals.monitoring.tcp.minecraft = { host = minecraftDomain; port = 25565; - location = "home"; + network = "internet"; + }; + globals.monitoring.http.minecraft-map = { + url = "https://${minecraftDomain}"; + expectedBodyRegex = "Minecraft Dynamic Map"; network = "internet"; }; @@ -400,6 +404,10 @@ in { zone minecraft 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Minecraft Dynamic Map"; + }; }; virtualHosts.${minecraftDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/paperless.nix b/hosts/sire/guests/paperless.nix index cefc220..0181bde 100644 --- a/hosts/sire/guests/paperless.nix +++ b/hosts/sire/guests/paperless.nix @@ -27,7 +27,7 @@ in { globals.services.paperless.domain = paperlessDomain; globals.monitoring.http.paperless = { url = "https://${paperlessDomain}"; - location = "home"; + expectedBodyRegex = "Paperless-ngx"; network = "internet"; }; @@ -39,6 +39,10 @@ in { zone paperless 64k; keepalive 2; ''; + # direct upstream monitoring doesn't work because + # paperless allowed hosts fails for ip-based queries. + # But that's fine, we just monitor it via the domain above anyway. + #monitoring.enable = true; }; virtualHosts.${paperlessDomain} = { forceSSL = true; @@ -63,6 +67,10 @@ in { zone paperless 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Paperless-ngx"; + }; }; virtualHosts.${paperlessDomain} = { forceSSL = true; diff --git a/hosts/sire/guests/samba.nix b/hosts/sire/guests/samba.nix index 44bda71..47b55e0 100644 --- a/hosts/sire/guests/samba.nix +++ b/hosts/sire/guests/samba.nix @@ -136,7 +136,6 @@ in { globals.monitoring.tcp.samba = { host = globals.net.home-lan.hosts.sire-samba.ipv4; port = 445; - location = "home"; network = "home-lan"; }; diff --git a/hosts/sire/net.nix b/hosts/sire/net.nix index 73c7a09..5bf6969 100644 --- a/hosts/sire/net.nix +++ b/hosts/sire/net.nix @@ -9,7 +9,6 @@ globals.monitoring.ping.sire = { hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv4; hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv6; - location = "home"; network = "home-lan"; }; diff --git a/hosts/ward/guests/adguardhome.nix b/hosts/ward/guests/adguardhome.nix index 4226738..b4d5f9a 100644 --- a/hosts/ward/guests/adguardhome.nix +++ b/hosts/ward/guests/adguardhome.nix @@ -16,14 +16,8 @@ in { globals.monitoring.dns.adguardhome = { server = globals.net.home-lan.hosts.ward-adguardhome.ipv4; domain = "."; - location = "home"; network = "home-lan"; }; - globals.monitoring.http.adguardhome = { - url = "https://${adguardhomeDomain}"; - location = "home"; - network = "internet"; - }; nodes.sentinel = { services.nginx = { @@ -33,6 +27,10 @@ in { zone adguardhome 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "AdGuard Home"; + }; }; virtualHosts.${adguardhomeDomain} = { forceSSL = true; diff --git a/hosts/ward/guests/forgejo.nix b/hosts/ward/guests/forgejo.nix index a0eac2b..a8b8b33 100644 --- a/hosts/ward/guests/forgejo.nix +++ b/hosts/ward/guests/forgejo.nix @@ -24,8 +24,8 @@ in { globals.services.forgejo.domain = forgejoDomain; globals.monitoring.http.forgejo = { - url = "https://${forgejoDomain}"; - location = "home"; + url = "https://${forgejoDomain}/user/login"; + expectedBodyRegex = "Redlew Git"; network = "internet"; }; @@ -57,6 +57,11 @@ in { zone forgejo 64k; keepalive 2; ''; + monitoring = { + enable = true; + path = "/user/login"; + expectedBodyRegex = "Redlew Git"; + }; }; virtualHosts.${forgejoDomain} = { forceSSL = true; diff --git a/hosts/ward/guests/kanidm.nix b/hosts/ward/guests/kanidm.nix index 031c9a5..ac87743 100644 --- a/hosts/ward/guests/kanidm.nix +++ b/hosts/ward/guests/kanidm.nix @@ -41,9 +41,10 @@ in { globals.services.kanidm.domain = kanidmDomain; globals.monitoring.http.kanidm = { - url = "https://${kanidmDomain}"; - location = "home"; + url = "https://${kanidmDomain}/status"; network = "internet"; + expectedBodyRegex = "true"; + skipTlsVerification = true; }; nodes.sentinel = { @@ -54,6 +55,13 @@ in { zone kanidm 64k; keepalive 2; ''; + monitoring = { + enable = true; + path = "/status"; + expectedBodyRegex = "true"; + skipTlsVerification = true; + useHttps = true; + }; }; virtualHosts.${kanidmDomain} = { forceSSL = true; diff --git a/hosts/ward/guests/netbird.nix b/hosts/ward/guests/netbird.nix index b6fda10..c24626e 100644 --- a/hosts/ward/guests/netbird.nix +++ b/hosts/ward/guests/netbird.nix @@ -79,8 +79,9 @@ in { globals.services.netbird.domain = netbirdDomain; globals.monitoring.http.netbird = { - url = "https://${netbirdDomain}"; - location = "home"; + url = "https://${netbirdDomain}/api/users"; + expectedStatus = 401; + expectedBodyRegex = "no valid authentication"; network = "internet"; }; @@ -92,6 +93,12 @@ in { zone netbird 64k; keepalive 5; ''; + monitoring = { + enable = true; + path = "/api/users"; + expectedStatus = 401; + expectedBodyRegex = "no valid authentication"; + }; }; upstreams.netbird-signal = { diff --git a/hosts/ward/guests/radicale.nix b/hosts/ward/guests/radicale.nix index 9d929c5..fe78ba8 100644 --- a/hosts/ward/guests/radicale.nix +++ b/hosts/ward/guests/radicale.nix @@ -9,7 +9,7 @@ in { globals.services.radicale.domain = radicaleDomain; globals.monitoring.http.radicale = { url = "https://${radicaleDomain}"; - location = "home"; + expectedBodyRegex = "Radicale Web Interface"; network = "internet"; }; @@ -21,6 +21,10 @@ in { zone radicale 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Radicale Web Interface"; + }; }; virtualHosts.${radicaleDomain} = { forceSSL = true; diff --git a/hosts/ward/guests/vaultwarden.nix b/hosts/ward/guests/vaultwarden.nix index c0d973a..f1f2c51 100644 --- a/hosts/ward/guests/vaultwarden.nix +++ b/hosts/ward/guests/vaultwarden.nix @@ -28,8 +28,7 @@ in { globals.services.vaultwarden.domain = vaultwardenDomain; globals.monitoring.http.vaultwarden = { url = "https://${vaultwardenDomain}"; - expectedBodyRegex = "Vaultwarden"; - location = "home"; + expectedBodyRegex = "Vaultwarden Web"; network = "internet"; }; @@ -41,6 +40,10 @@ in { zone vaultwarden 64k; keepalive 2; ''; + monitoring = { + enable = true; + expectedBodyRegex = "Vaultwarden Web"; + }; }; virtualHosts.${vaultwardenDomain} = { forceSSL = true; diff --git a/hosts/ward/guests/web-proxy.nix b/hosts/ward/guests/web-proxy.nix index ae18c4c..cba4861 100644 --- a/hosts/ward/guests/web-proxy.nix +++ b/hosts/ward/guests/web-proxy.nix @@ -11,6 +11,9 @@ in { firewallRuleForAll.allowedTCPPorts = [80 443]; }; + # This node shall monitor the infrastructure + meta.telegraf.availableMonitoringNetworks = ["internet" "home-wan" "home-lan"]; + age.secrets.acme-cloudflare-dns-token = { rekeyFile = config.node.secretsDir + "/acme-cloudflare-dns-token.age"; mode = "440"; diff --git a/hosts/ward/net.nix b/hosts/ward/net.nix index 7968e3f..9fb14e6 100644 --- a/hosts/ward/net.nix +++ b/hosts/ward/net.nix @@ -9,7 +9,6 @@ globals.monitoring.ping.ward = { hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv4; hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv6; - location = "home"; network = "home-lan"; }; diff --git a/hosts/zackbiene/net.nix b/hosts/zackbiene/net.nix index bd35be3..0b3abad 100644 --- a/hosts/zackbiene/net.nix +++ b/hosts/zackbiene/net.nix @@ -12,7 +12,6 @@ in { globals.monitoring.ping.zackbiene = { hostv4 = "zackbiene.local"; hostv6 = "zackbiene.local"; - location = "home"; network = "home-lan"; }; diff --git a/modules/default.nix b/modules/default.nix index f8a6624..223a3a5 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -13,6 +13,7 @@ ./kanidm.nix ./meta.nix ./netbird-client.nix + ./nginx-upstream-monitoring.nix ./oauth2-proxy.nix ./promtail.nix ./secrets.nix diff --git a/modules/globals.nix b/modules/globals.nix index 1dd2cbe..0c7e723 100644 --- a/modules/globals.nix +++ b/modules/globals.nix @@ -8,6 +8,13 @@ mkOption types ; + + defaultOptions = { + network = mkOption { + type = types.str; + description = "The network to which this endpoint is associated."; + }; + }; in { options = { globals = mkOption { @@ -97,120 +104,94 @@ in { monitoring = { ping = mkOption { type = types.attrsOf (types.submodule { - options = { - hostv4 = mkOption { - type = types.nullOr types.str; - description = "The IP/hostname to ping via ipv4."; - default = null; - }; + options = + defaultOptions + // { + hostv4 = mkOption { + type = types.nullOr types.str; + description = "The IP/hostname to ping via ipv4."; + default = null; + }; - hostv6 = mkOption { - type = types.nullOr types.str; - description = "The IP/hostname to ping via ipv6."; - default = null; + hostv6 = mkOption { + type = types.nullOr types.str; + description = "The IP/hostname to ping via ipv6."; + default = null; + }; }; - - location = mkOption { - type = types.str; - description = "A location tag added to this metric."; - }; - - network = mkOption { - type = types.str; - description = "The network to which this endpoint is associated."; - }; - }; }); }; http = mkOption { type = types.attrsOf (types.submodule { - options = { - url = mkOption { - type = types.str; - description = "The url to connect to."; - }; + options = + defaultOptions + // { + url = mkOption { + type = types.either (types.listOf types.str) types.str; + description = "The url to connect to."; + }; - location = mkOption { - type = types.str; - description = "A location tag added to this metric."; - }; + expectedStatus = mkOption { + type = types.int; + default = 200; + description = "The HTTP status code to expect."; + }; - network = mkOption { - type = types.str; - description = "The network to which this endpoint is associated."; - }; + expectedBodyRegex = mkOption { + type = types.nullOr types.str; + description = "A regex pattern to expect in the body."; + default = null; + }; - expectedStatus = mkOption { - type = types.int; - default = 200; - description = "The HTTP status code to expect."; + skipTlsVerification = mkOption { + type = types.bool; + description = "Skip tls verification when using https."; + default = false; + }; }; - - expectedBodyRegex = mkOption { - type = types.nullOr types.str; - description = "A regex pattern to expect in the body."; - default = null; - }; - }; }); }; dns = mkOption { type = types.attrsOf (types.submodule { - options = { - server = mkOption { - type = types.str; - description = "The DNS server to query."; - }; + options = + defaultOptions + // { + server = mkOption { + type = types.str; + description = "The DNS server to query."; + }; - domain = mkOption { - type = types.str; - description = "The domain to query."; - }; + domain = mkOption { + type = types.str; + description = "The domain to query."; + }; - record-type = mkOption { - type = types.str; - description = "The record type to query."; - default = "A"; + record-type = mkOption { + type = types.str; + description = "The record type to query."; + default = "A"; + }; }; - - location = mkOption { - type = types.str; - description = "A location tag added to this metric."; - }; - - network = mkOption { - type = types.str; - description = "The network to which this endpoint is associated."; - }; - }; }); }; tcp = mkOption { type = types.attrsOf (types.submodule { - options = { - host = mkOption { - type = types.str; - description = "The IP/hostname to connect to."; - }; + options = + defaultOptions + // { + host = mkOption { + type = types.str; + description = "The IP/hostname to connect to."; + }; - port = mkOption { - type = types.port; - description = "The port to connect to."; + port = mkOption { + type = types.port; + description = "The port to connect to."; + }; }; - - location = mkOption { - type = types.str; - description = "A location tag added to this metric."; - }; - - network = mkOption { - type = types.str; - description = "The network to which this endpoint is associated."; - }; - }; }); }; }; diff --git a/modules/nginx-upstream-monitoring.nix b/modules/nginx-upstream-monitoring.nix new file mode 100644 index 0000000..eb3dd53 --- /dev/null +++ b/modules/nginx-upstream-monitoring.nix @@ -0,0 +1,81 @@ +{ + config, + lib, + ... +}: let + inherit + (lib) + attrNames + filterAttrs + flip + mapAttrs' + mkOption + nameValuePair + types + ; +in { + options.services.nginx.upstreams = mkOption { + type = types.attrsOf (types.submodule { + options.monitoring = { + enable = mkOption { + type = types.bool; + description = "Whether to add a global monitoring entry for this upstream"; + default = false; + }; + + path = mkOption { + type = types.str; + description = "The path to query."; + default = ""; + }; + + expectedStatus = mkOption { + type = types.int; + default = 200; + description = "The HTTP status code to expect."; + }; + + expectedBodyRegex = mkOption { + type = types.nullOr types.str; + description = "A regex pattern to expect in the body."; + default = null; + }; + + useHttps = mkOption { + type = types.bool; + description = "Whether to use https to connect to this upstream when monitoring"; + default = false; + }; + + skipTlsVerification = mkOption { + type = types.bool; + description = "Skip tls verification when using https."; + default = false; + }; + }; + }); + }; + + config = let + monitoredUpstreams = filterAttrs (_: x: x.monitoring.enable) config.services.nginx.upstreams; + in { + globals.monitoring.http = flip mapAttrs' monitoredUpstreams ( + upstreamName: upstream: let + schema = + if upstream.monitoring.useHttps + then "https" + else "http"; + in + nameValuePair "${config.node.name}-upstream-${upstreamName}" { + url = map (server: "${schema}://${server}${upstream.monitoring.path}") (attrNames upstream.servers); + network = "local-${config.node.name}"; + inherit + (upstream.monitoring) + expectedBodyRegex + expectedStatus + skipTlsVerification + ; + } + ); + }; +} diff --git a/modules/telegraf.nix b/modules/telegraf.nix index 7a35392..0cf9370 100644 --- a/modules/telegraf.nix +++ b/modules/telegraf.nix @@ -21,6 +21,7 @@ optional optionalAttrs optionals + toList types ; @@ -44,16 +45,14 @@ in { description = "Additional secrets to replace in pre-start. The attr name will be searched and replaced in the config with the value read from the given file."; }; - globalMonitoring = { - enable = mkEnableOption "monitor the global infrastructure from this node."; - availableNetworks = mkOption { - type = types.listOf types.str; - example = ["internet"]; - description = '' - The networks that can be reached from this node. - Only global entries with a matching network will be monitored from here. - ''; - }; + availableMonitoringNetworks = mkOption { + type = types.listOf types.str; + example = ["internet"]; + description = '' + Any of the global monitoring definitions which has a network from this list + will automatically be monitored via telegraf. Set this to any networks that + can be reached from this node. This includes `local-` by default. + ''; }; influxdb2 = { @@ -87,6 +86,9 @@ in { }; config = mkIf (!minimal && cfg.enable) { + # Monitor anything that can only be monitored from this node + meta.telegraf.availableMonitoringNetworks = ["local-${config.node.name}"]; + assertions = [ { assertion = !config.boot.isContainer; @@ -185,6 +187,75 @@ in { }; temp = {}; wireguard = {}; + + ping = concatLists (flip mapAttrsToList globals.monitoring.ping ( + name: pingCfg: + optionals (elem pingCfg.network cfg.availableMonitoringNetworks) ( + concatLists (forEach ["hostv4" "hostv6"] ( + attr: + optional (pingCfg.${attr} != null) { + method = "native"; + urls = [pingCfg.${attr}]; + ipv4 = attr == "hostv4"; + ipv6 = attr == "hostv6"; + tags = { + inherit name; + inherit (pingCfg) network; + ip_version = + if attr == "hostv4" + then "v4" + else "v6"; + }; + fieldinclude = [ + "percent_packet_loss" + "average_response_ms" + ]; + } + )) + ) + )); + + http_response = concatLists (flip mapAttrsToList globals.monitoring.http ( + name: httpCfg: + optional (elem httpCfg.network cfg.availableMonitoringNetworks) { + urls = toList httpCfg.url; + method = "GET"; + response_status_code = httpCfg.expectedStatus; + response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex; + insecure_skip_verify = httpCfg.skipTlsVerification; + follow_redirects = true; + tags = { + inherit name; + inherit (httpCfg) network; + }; + } + )); + + dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns ( + name: dnsCfg: + optional (elem dnsCfg.network cfg.availableMonitoringNetworks) { + servers = [dnsCfg.server]; + domains = [dnsCfg.domain]; + record_type = dnsCfg.record-type; + tags = { + inherit name; + inherit (dnsCfg) network; + }; + } + )); + + net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp ( + name: tcpCfg: + optional (elem tcpCfg.network cfg.availableMonitoringNetworks) { + address = "${tcpCfg.host}:${toString tcpCfg.port}"; + protocol = "tcp"; + tags = { + inherit name; + inherit (tcpCfg) network; + }; + fieldexclude = ["result_type" "string_found"]; + } + )); } // optionalAttrs config.services.smartd.enable { sensors = {}; @@ -200,74 +271,6 @@ in { } // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) { wireless = {}; - } - // optionalAttrs cfg.globalMonitoring.enable { - ping = concatLists (flip mapAttrsToList globals.monitoring.ping ( - name: pingCfg: - optionals (elem pingCfg.network cfg.globalMonitoring.availableNetworks) ( - concatLists (forEach ["hostv4" "hostv6"] ( - attr: - optional (pingCfg.${attr} != null) { - method = "native"; - urls = [pingCfg.${attr}]; - ipv4 = attr == "hostv4"; - ipv6 = attr == "hostv6"; - tags = { - inherit name; - inherit (pingCfg) location network; - ip_version = - if attr == "hostv4" - then "v4" - else "v6"; - }; - fieldpass = [ - "percent_packet_loss" - "average_response_ms" - ]; - } - )) - ) - )); - - http_response = concatLists (flip mapAttrsToList globals.monitoring.http ( - name: httpCfg: - optional (elem httpCfg.network cfg.globalMonitoring.availableNetworks) { - urls = [httpCfg.url]; - method = "GET"; - response_status_code = httpCfg.expectedStatus; - response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex; - tags = { - inherit name; - inherit (httpCfg) location network; - }; - } - )); - - dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns ( - name: dnsCfg: - optional (elem dnsCfg.network cfg.globalMonitoring.availableNetworks) { - servers = [dnsCfg.server]; - domains = [dnsCfg.domain]; - record_type = dnsCfg.record-type; - tags = { - inherit name; - inherit (dnsCfg) location network; - }; - } - )); - - net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp ( - name: tcpCfg: - optional (elem tcpCfg.network cfg.globalMonitoring.availableNetworks) { - address = "${tcpCfg.host}:${toString tcpCfg.port}"; - protocol = "tcp"; - tags = { - inherit name; - inherit (tcpCfg) location network; - }; - fieldexclude = ["result_type" "string_found"]; - } - )); }; }; };