diff --git a/globals.nix b/globals.nix new file mode 100644 index 0000000..9283bfe --- /dev/null +++ b/globals.nix @@ -0,0 +1,59 @@ +{config, ...}: let + inherit (config) globals; +in { + globals = { + net = { + home-wan = { + cidrv4 = "192.168.178.0/24"; + hosts.fritzbox.id = 1; + hosts.ward.id = 2; + }; + + home-lan = { + cidrv4 = "192.168.1.0/24"; + cidrv6 = "fd10::/64"; + hosts.ward.id = 1; + hosts.sire.id = 2; + hosts.ward-adguardhome.id = 3; + hosts.ward-web-proxy.id = 4; + hosts.sire-samba.id = 10; + }; + + proxy-home = { + cidrv4 = "10.44.0.0/24"; + cidrv6 = "fd00:44::/120"; + }; + }; + + monitoring = { + dns.cloudflare = { + server = "1.1.1.1"; + domain = "."; + location = "home"; + network = "home-lan"; + }; + + ping = { + cloudflare = { + hostv4 = "1.1.1.1"; + hostv6 = "2606:4700:4700::1111"; + location = "external"; + network = "internet"; + }; + + google = { + hostv4 = "8.8.8.8"; + hostv6 = "2001:4860:4860::8888"; + location = "external"; + network = "internet"; + }; + + fritz-box = { + hostv4 = globals.net.home-wan.hosts.fritzbox.ipv4; + location = "home"; + network = "home-wan"; + }; + }; + }; + }; +} diff --git a/hosts/envoy/net.nix b/hosts/envoy/net.nix index 37d4248..d10a1a2 100644 --- a/hosts/envoy/net.nix +++ b/hosts/envoy/net.nix @@ -1,17 +1,28 @@ -{config, ...}: { +{ + config, + lib, + ... +}: let + icfg = config.repo.secrets.local.networking.interfaces.wan; +in { networking.hostId = config.repo.secrets.local.networking.hostId; networking.domain = config.repo.secrets.global.domains.mail.primary; networking.hosts."127.0.0.1" = ["mx1.${config.repo.secrets.global.domains.mail.primary}"]; + globals.monitoring.ping.envoy = { + hostv4 = lib.net.cidr.ip icfg.hostCidrv4; + hostv6 = lib.net.cidr.ip icfg.hostCidrv6; + location = "external"; + network = "internet"; + }; + boot.initrd.systemd.network = { enable = true; networks = {inherit (config.systemd.network.networks) "10-wan";}; }; systemd.network.networks = { - "10-wan" = let - icfg = config.repo.secrets.local.networking.interfaces.wan; - in { + "10-wan" = { address = [ icfg.hostCidrv4 icfg.hostCidrv6 diff --git a/hosts/sentinel/net.nix b/hosts/sentinel/net.nix index 3ac0573..519b0f4 100644 --- a/hosts/sentinel/net.nix +++ b/hosts/sentinel/net.nix @@ -1,7 +1,20 @@ -{config, ...}: { +{ + config, + lib, + ... +}: let + icfg = config.repo.secrets.local.networking.interfaces.wan; +in { networking.hostId = config.repo.secrets.local.networking.hostId; networking.domain = config.repo.secrets.global.domains.me; + globals.monitoring.ping.sentinel = { + hostv4 = lib.net.cidr.ip icfg.hostCidrv4; + hostv6 = lib.net.cidr.ip icfg.hostCidrv6; + location = "external"; + network = "internet"; + }; + # Forwarding required for forgejo 9922->22 boot.kernel.sysctl."net.ipv4.ip_forward" = 1; @@ -11,9 +24,7 @@ }; systemd.network.networks = { - "10-wan" = let - icfg = config.repo.secrets.local.networking.interfaces.wan; - in { + "10-wan" = { address = [ icfg.hostCidrv4 icfg.hostCidrv6 diff --git a/hosts/sire/guests/ai.nix b/hosts/sire/guests/ai.nix index f60302d..679da2f 100644 --- a/hosts/sire/guests/ai.nix +++ b/hosts/sire/guests/ai.nix @@ -51,6 +51,12 @@ in { }; globals.services.open-webui.domain = openWebuiDomain; + globals.monitoring.http.ollama-webui = { + url = "https://${openWebuiDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.open-webui = { diff --git a/hosts/sire/guests/grafana.nix b/hosts/sire/guests/grafana.nix index 9493dcf..845faed 100644 --- a/hosts/sire/guests/grafana.nix +++ b/hosts/sire/guests/grafana.nix @@ -78,6 +78,12 @@ in { }; globals.services.grafana.domain = grafanaDomain; + globals.monitoring.http.grafana = { + url = "https://${grafanaDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { age.secrets.loki-basic-auth-hashes.generator.dependencies = [ config.age.secrets.grafana-loki-basic-auth-password diff --git a/hosts/sire/guests/immich.nix b/hosts/sire/guests/immich.nix index fbd3a2f..6ea9fa3 100644 --- a/hosts/sire/guests/immich.nix +++ b/hosts/sire/guests/immich.nix @@ -191,6 +191,12 @@ in { }; globals.services.immich.domain = immichDomain; + globals.monitoring.http.immich = { + url = "https://${immichDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.immich = { diff --git a/hosts/sire/guests/influxdb.nix b/hosts/sire/guests/influxdb.nix index 95d462e..7a3448c 100644 --- a/hosts/sire/guests/influxdb.nix +++ b/hosts/sire/guests/influxdb.nix @@ -28,58 +28,13 @@ in { }; meta.telegraf.secrets."@GITHUB_ACCESS_TOKEN@" = config.age.secrets.github-access-token.path; + meta.telegraf.globalMonitoring = { + enable = true; + availableNetworks = ["internet" "home-wan" "home-lan"]; + }; services.telegraf.extraConfig.outputs.influxdb_v2.urls = lib.mkForce ["http://localhost:${toString influxdbPort}"]; - globals.monitoring.ping.cloudflare-dns = { - host = "1.1.1.1"; - location = "external"; - }; - - globals.monitoring.ping.google-dns = { - host = "8.8.8.8"; - location = "external"; - }; - services.telegraf.extraConfig.inputs = { - ping = [ - { - method = "native"; - urls = [ - globals.net.home-wan.hosts.fritzbox.ipv4 - globals.net.home-lan.hosts.ward.ipv4 - ]; - tags.type = "internal"; - fieldpass = [ - "percent_packet_loss" - "average_response_ms" - ]; - } - { - method = "native"; - urls = [ - "1.1.1.1" - "8.8.8.8" - config.repo.secrets.global.domains.me - config.repo.secrets.global.domains.personal - ]; - tags.type = "external"; - fieldpass = [ - "percent_packet_loss" - "average_response_ms" - ]; - } - ]; - - # FIXME: pls define this on the relevant hosts. Then we can ping it from multiple other hosts - #http_response = [ - # { - # urls = [ - # ]; - # response_string_match = "Index of /"; - # response_status_code = 200; - # } - #]; - github = { access_token = "@GITHUB_ACCESS_TOKEN@"; repositories = [ @@ -94,6 +49,12 @@ in { }; globals.services.influxdb.domain = influxdbDomain; + globals.monitoring.http.influxdb = { + url = "https://${influxdbDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.influxdb = { diff --git a/hosts/sire/guests/loki.nix b/hosts/sire/guests/loki.nix index cb724e7..ec43a38 100644 --- a/hosts/sire/guests/loki.nix +++ b/hosts/sire/guests/loki.nix @@ -18,6 +18,12 @@ in { }; globals.services.loki.domain = lokiDomain; + globals.monitoring.http.loki = { + url = "https://${lokiDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { age.secrets.loki-basic-auth-hashes = { generator.script = "basic-auth"; diff --git a/hosts/sire/guests/minecraft.nix b/hosts/sire/guests/minecraft.nix index 971b9c8..7f4ce2a 100644 --- a/hosts/sire/guests/minecraft.nix +++ b/hosts/sire/guests/minecraft.nix @@ -1,3 +1,4 @@ +# FIXME: todo: host the proxy on sentinel so the IPs are not lost in natting { config, pkgs, @@ -360,6 +361,13 @@ in { ]; globals.services.minecraft.domain = minecraftDomain; + globals.monitoring.tcp.minecraft = { + host = minecraftDomain; + port = 25565; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { # Rewrite destination addr with dnat on incoming connections # and masquerade responses to make them look like they originate from this host. diff --git a/hosts/sire/guests/paperless.nix b/hosts/sire/guests/paperless.nix index 9131e41..cefc220 100644 --- a/hosts/sire/guests/paperless.nix +++ b/hosts/sire/guests/paperless.nix @@ -25,6 +25,12 @@ in { }; globals.services.paperless.domain = paperlessDomain; + globals.monitoring.http.paperless = { + url = "https://${paperlessDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.paperless = { diff --git a/hosts/sire/guests/samba.nix b/hosts/sire/guests/samba.nix index ebb470c..44bda71 100644 --- a/hosts/sire/guests/samba.nix +++ b/hosts/sire/guests/samba.nix @@ -133,6 +133,13 @@ in { openFirewall = true; }; + globals.monitoring.tcp.samba = { + host = globals.net.home-lan.hosts.sire-samba.ipv4; + port = 445; + location = "home"; + network = "home-lan"; + }; + services.samba = { enable = true; openFirewall = true; diff --git a/hosts/sire/net.nix b/hosts/sire/net.nix index 76e04c3..73c7a09 100644 --- a/hosts/sire/net.nix +++ b/hosts/sire/net.nix @@ -1,10 +1,18 @@ { config, globals, + lib, ... }: { networking.hostId = config.repo.secrets.local.networking.hostId; + globals.monitoring.ping.sire = { + hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv4; + hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv6; + location = "home"; + network = "home-lan"; + }; + boot.initrd.systemd.network = { enable = true; networks."10-lan" = { diff --git a/hosts/ward/guests/adguardhome.nix b/hosts/ward/guests/adguardhome.nix index 76657ac..4226738 100644 --- a/hosts/ward/guests/adguardhome.nix +++ b/hosts/ward/guests/adguardhome.nix @@ -13,6 +13,18 @@ in { }; globals.services.adguardhome.domain = adguardhomeDomain; + globals.monitoring.dns.adguardhome = { + server = globals.net.home-lan.hosts.ward-adguardhome.ipv4; + domain = "."; + location = "home"; + network = "home-lan"; + }; + globals.monitoring.http.adguardhome = { + url = "https://${adguardhomeDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.adguardhome = { diff --git a/hosts/ward/guests/forgejo.nix b/hosts/ward/guests/forgejo.nix index b2789fb..a0eac2b 100644 --- a/hosts/ward/guests/forgejo.nix +++ b/hosts/ward/guests/forgejo.nix @@ -23,6 +23,12 @@ in { }; globals.services.forgejo.domain = forgejoDomain; + globals.monitoring.http.forgejo = { + url = "https://${forgejoDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { # Rewrite destination addr with dnat on incoming connections # and masquerade responses to make them look like they originate from this host. diff --git a/hosts/ward/guests/kanidm.nix b/hosts/ward/guests/kanidm.nix index 344f26d..031c9a5 100644 --- a/hosts/ward/guests/kanidm.nix +++ b/hosts/ward/guests/kanidm.nix @@ -40,6 +40,12 @@ in { age.secrets.kanidm-oauth2-web-sentinel = mkRandomSecret; globals.services.kanidm.domain = kanidmDomain; + globals.monitoring.http.kanidm = { + url = "https://${kanidmDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.kanidm = { diff --git a/hosts/ward/guests/netbird.nix b/hosts/ward/guests/netbird.nix index 122a4b7..b6fda10 100644 --- a/hosts/ward/guests/netbird.nix +++ b/hosts/ward/guests/netbird.nix @@ -78,6 +78,12 @@ in { }; globals.services.netbird.domain = netbirdDomain; + globals.monitoring.http.netbird = { + url = "https://${netbirdDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.netbird-mgmt = { diff --git a/hosts/ward/guests/radicale.nix b/hosts/ward/guests/radicale.nix index f214fc8..9d929c5 100644 --- a/hosts/ward/guests/radicale.nix +++ b/hosts/ward/guests/radicale.nix @@ -7,6 +7,12 @@ in { }; globals.services.radicale.domain = radicaleDomain; + globals.monitoring.http.radicale = { + url = "https://${radicaleDomain}"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.radicale = { diff --git a/hosts/ward/guests/vaultwarden.nix b/hosts/ward/guests/vaultwarden.nix index 3b1587b..c0d973a 100644 --- a/hosts/ward/guests/vaultwarden.nix +++ b/hosts/ward/guests/vaultwarden.nix @@ -26,6 +26,13 @@ in { ]; globals.services.vaultwarden.domain = vaultwardenDomain; + globals.monitoring.http.vaultwarden = { + url = "https://${vaultwardenDomain}"; + expectedBodyRegex = "Vaultwarden"; + location = "home"; + network = "internet"; + }; + nodes.sentinel = { services.nginx = { upstreams.vaultwarden = { diff --git a/hosts/ward/net.nix b/hosts/ward/net.nix index b3f450d..7968e3f 100644 --- a/hosts/ward/net.nix +++ b/hosts/ward/net.nix @@ -1,31 +1,16 @@ { config, globals, + lib, ... }: { networking.hostId = config.repo.secrets.local.networking.hostId; - globals.net = { - home-wan = { - cidrv4 = "192.168.178.0/24"; - hosts.fritzbox.id = 1; - hosts.ward.id = 2; - }; - - home-lan = { - cidrv4 = "192.168.1.0/24"; - cidrv6 = "fd10::/64"; - hosts.ward.id = 1; - hosts.sire.id = 2; - hosts.ward-adguardhome.id = 3; - hosts.ward-web-proxy.id = 4; - hosts.sire-samba.id = 10; - }; - - proxy-home = { - cidrv4 = "10.44.0.0/24"; - cidrv6 = "fd00:44::/120"; - }; + globals.monitoring.ping.ward = { + hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv4; + hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv6; + location = "home"; + network = "home-lan"; }; boot.initrd.systemd.network = { diff --git a/hosts/zackbiene/net.nix b/hosts/zackbiene/net.nix index fee1b8f..bd35be3 100644 --- a/hosts/zackbiene/net.nix +++ b/hosts/zackbiene/net.nix @@ -9,6 +9,13 @@ in { networking.hostId = config.repo.secrets.local.networking.hostId; + globals.monitoring.ping.zackbiene = { + hostv4 = "zackbiene.local"; + hostv6 = "zackbiene.local"; + location = "home"; + network = "home-lan"; + }; + wireguard.proxy-home.client.via = "ward"; boot.initrd.systemd.network = { diff --git a/modules/globals.nix b/modules/globals.nix index ad45d17..1dd2cbe 100644 --- a/modules/globals.nix +++ b/modules/globals.nix @@ -98,15 +98,118 @@ in { ping = mkOption { type = types.attrsOf (types.submodule { options = { - host = mkOption { - type = types.str; - description = "The IP/hostname to ping."; + hostv4 = mkOption { + type = types.nullOr types.str; + description = "The IP/hostname to ping via ipv4."; + default = null; + }; + + hostv6 = mkOption { + type = types.nullOr types.str; + description = "The IP/hostname to ping via ipv6."; + default = null; }; location = mkOption { type = types.str; description = "A location tag added to this metric."; }; + + network = mkOption { + type = types.str; + description = "The network to which this endpoint is associated."; + }; + }; + }); + }; + + http = mkOption { + type = types.attrsOf (types.submodule { + options = { + url = mkOption { + type = types.str; + description = "The url to connect to."; + }; + + location = mkOption { + type = types.str; + description = "A location tag added to this metric."; + }; + + network = mkOption { + type = types.str; + description = "The network to which this endpoint is associated."; + }; + + expectedStatus = mkOption { + type = types.int; + default = 200; + description = "The HTTP status code to expect."; + }; + + expectedBodyRegex = mkOption { + type = types.nullOr types.str; + description = "A regex pattern to expect in the body."; + default = null; + }; + }; + }); + }; + + dns = mkOption { + type = types.attrsOf (types.submodule { + options = { + server = mkOption { + type = types.str; + description = "The DNS server to query."; + }; + + domain = mkOption { + type = types.str; + description = "The domain to query."; + }; + + record-type = mkOption { + type = types.str; + description = "The record type to query."; + default = "A"; + }; + + location = mkOption { + type = types.str; + description = "A location tag added to this metric."; + }; + + network = mkOption { + type = types.str; + description = "The network to which this endpoint is associated."; + }; + }; + }); + }; + + tcp = mkOption { + type = types.attrsOf (types.submodule { + options = { + host = mkOption { + type = types.str; + description = "The IP/hostname to connect to."; + }; + + port = mkOption { + type = types.port; + description = "The port to connect to."; + }; + + location = mkOption { + type = types.str; + description = "A location tag added to this metric."; + }; + + network = mkOption { + type = types.str; + description = "The network to which this endpoint is associated."; + }; }; }); }; diff --git a/modules/telegraf.nix b/modules/telegraf.nix index bfad346..7a35392 100644 --- a/modules/telegraf.nix +++ b/modules/telegraf.nix @@ -1,5 +1,6 @@ { config, + globals, lib, minimal, nodes, @@ -8,11 +9,18 @@ }: let inherit (lib) + concatLists + elem + flip + forEach + mapAttrsToList mkAfter mkEnableOption mkIf mkOption + optional optionalAttrs + optionals types ; @@ -36,6 +44,18 @@ in { description = "Additional secrets to replace in pre-start. The attr name will be searched and replaced in the config with the value read from the given file."; }; + globalMonitoring = { + enable = mkEnableOption "monitor the global infrastructure from this node."; + availableNetworks = mkOption { + type = types.listOf types.str; + example = ["internet"]; + description = '' + The networks that can be reached from this node. + Only global entries with a matching network will be monitored from here. + ''; + }; + }; + influxdb2 = { domain = mkOption { type = types.str; @@ -165,8 +185,6 @@ in { }; temp = {}; wireguard = {}; - # http_response = { urls = [ "http://localhost/" ]; }; - # ping = { urls = [ "9.9.9.9" ]; }; } // optionalAttrs config.services.smartd.enable { sensors = {}; @@ -182,6 +200,74 @@ in { } // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) { wireless = {}; + } + // optionalAttrs cfg.globalMonitoring.enable { + ping = concatLists (flip mapAttrsToList globals.monitoring.ping ( + name: pingCfg: + optionals (elem pingCfg.network cfg.globalMonitoring.availableNetworks) ( + concatLists (forEach ["hostv4" "hostv6"] ( + attr: + optional (pingCfg.${attr} != null) { + method = "native"; + urls = [pingCfg.${attr}]; + ipv4 = attr == "hostv4"; + ipv6 = attr == "hostv6"; + tags = { + inherit name; + inherit (pingCfg) location network; + ip_version = + if attr == "hostv4" + then "v4" + else "v6"; + }; + fieldpass = [ + "percent_packet_loss" + "average_response_ms" + ]; + } + )) + ) + )); + + http_response = concatLists (flip mapAttrsToList globals.monitoring.http ( + name: httpCfg: + optional (elem httpCfg.network cfg.globalMonitoring.availableNetworks) { + urls = [httpCfg.url]; + method = "GET"; + response_status_code = httpCfg.expectedStatus; + response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex; + tags = { + inherit name; + inherit (httpCfg) location network; + }; + } + )); + + dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns ( + name: dnsCfg: + optional (elem dnsCfg.network cfg.globalMonitoring.availableNetworks) { + servers = [dnsCfg.server]; + domains = [dnsCfg.domain]; + record_type = dnsCfg.record-type; + tags = { + inherit name; + inherit (dnsCfg) location network; + }; + } + )); + + net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp ( + name: tcpCfg: + optional (elem tcpCfg.network cfg.globalMonitoring.availableNetworks) { + address = "${tcpCfg.host}:${toString tcpCfg.port}"; + protocol = "tcp"; + tags = { + inherit name; + inherit (tcpCfg) location network; + }; + fieldexclude = ["result_type" "string_found"]; + } + )); }; }; }; diff --git a/nix/globals.nix b/nix/globals.nix index 30c59cb..450ed1e 100644 --- a/nix/globals.nix +++ b/nix/globals.nix @@ -12,6 +12,7 @@ }; modules = [ ../modules/globals.nix + ../globals.nix ({lib, ...}: { globals = lib.mkMerge ( lib.concatLists (lib.flip lib.mapAttrsToList config.nodes (