mirrors_public/oddlama_nix-config
1
1
Fork 1
mirror of https://github.com/oddlama/nix-config.git synced 2025-10-10 14:50:40 +02:00
Code Activity

feat(monitoring): remove location, add nginx upstream monitoring option

Browse source
This commit is contained in:
oddlama 2024-07-15 17:36:04 +02:00
parent 2024c3bfd5
commit 18b2002c27
No known key found for this signature in database
GPG key ID: 14EFE510775FE39A
26 changed files with 352 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

20
globals.nix
View file

@ -28,31 +28,35 @@ in {
};
monitoring = {
dns.cloudflare = {
server = "1.1.1.1";
domain = ".";
location = "home";
network = "home-lan";
dns = {
cloudflare = {
server = "1.1.1.1";
domain = ".";
network = "internet";
};
google = {
server = "8.8.8.8";
domain = ".";
network = "internet";
};
};
ping = {
cloudflare = {
hostv4 = "1.1.1.1";
hostv6 = "2606:4700:4700::1111";
location = "external";
network = "internet";
};
google = {
hostv4 = "8.8.8.8";
hostv6 = "2001:4860:4860::8888";
location = "external";
network = "internet";
};
fritz-box = {
hostv4 = globals.net.home-wan.hosts.fritzbox.ipv4;
location = "home";
network = "home-wan";
};
};

1
hosts/envoy/net.nix
View file

@ -12,7 +12,6 @@ in {
globals.monitoring.ping.envoy = {
hostv4 = lib.net.cidr.ip icfg.hostCidrv4;
hostv6 = lib.net.cidr.ip icfg.hostCidrv6;
location = "external";
network = "internet";
};

3
hosts/sentinel/default.nix
View file

@ -55,5 +55,8 @@
bucket = "telegraf";
node = "sire-influxdb";
};
# This node shall monitor the infrastructure
availableMonitoringNetworks = ["internet"];
};
}

1
hosts/sentinel/net.nix
View file

@ -11,7 +11,6 @@ in {
globals.monitoring.ping.sentinel = {
hostv4 = lib.net.cidr.ip icfg.hostCidrv4;
hostv6 = lib.net.cidr.ip icfg.hostCidrv6;
location = "external";
network = "internet";
};

12
hosts/sire/guests/ai.nix
View file

@ -51,10 +51,10 @@ in {
};
globals.services.open-webui.domain = openWebuiDomain;
globals.monitoring.http.ollama-webui = {
url = "https://${openWebuiDomain}";
location = "home";
network = "internet";
globals.monitoring.http.ollama = {
url = config.services.open-webui.environment.OLLAMA_BASE_URL;
expectedBodyRegex = "Ollama is running";
network = "local-${config.node.name}";
};
nodes.sentinel = {
@ -65,6 +65,10 @@ in {
zone open-webui 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Open WebUI";
};
};
virtualHosts.${openWebuiDomain} = {
forceSSL = true;

10
hosts/sire/guests/grafana.nix
View file

@ -80,7 +80,7 @@ in {
globals.services.grafana.domain = grafanaDomain;
globals.monitoring.http.grafana = {
url = "https://${grafanaDomain}";
location = "home";
expectedBodyRegex = "Grafana";
network = "internet";
};
@ -96,6 +96,10 @@ in {
zone grafana 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Grafana";
};
};
virtualHosts.${grafanaDomain} = {
forceSSL = true;
@ -116,6 +120,10 @@ in {
zone grafana 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Grafana";
};
};
virtualHosts.${grafanaDomain} = {
forceSSL = true;

10
hosts/sire/guests/immich.nix
View file

@ -193,7 +193,7 @@ in {
globals.services.immich.domain = immichDomain;
globals.monitoring.http.immich = {
url = "https://${immichDomain}";
location = "home";
expectedBodyRegex = "immutable.entry.app";
network = "internet";
};
@ -205,6 +205,10 @@ in {
zone immich 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "immutable.entry.app";
};
};
virtualHosts.${immichDomain} = {
forceSSL = true;
@ -228,6 +232,10 @@ in {
zone immich 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "immutable.entry.app";
};
};
virtualHosts.${immichDomain} = {
forceSSL = true;

18
hosts/sire/guests/influxdb.nix
View file

@ -1,6 +1,5 @@
{
config,
globals,
lib,
nodes,
pkgs,
@ -28,10 +27,6 @@ in {
};
meta.telegraf.secrets."@GITHUB_ACCESS_TOKEN@" = config.age.secrets.github-access-token.path;
meta.telegraf.globalMonitoring = {
enable = true;
availableNetworks = ["internet" "home-wan" "home-lan"];
};
services.telegraf.extraConfig.outputs.influxdb_v2.urls = lib.mkForce ["http://localhost:${toString influxdbPort}"];
services.telegraf.extraConfig.inputs = {
@ -49,11 +44,6 @@ in {
};
globals.services.influxdb.domain = influxdbDomain;
globals.monitoring.http.influxdb = {
url = "https://${influxdbDomain}";
location = "home";
network = "internet";
};
nodes.sentinel = {
services.nginx = {
@ -63,6 +53,10 @@ in {
zone influxdb 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "InfluxDB";
};
};
virtualHosts.${influxdbDomain} = let
accessRules = ''
@ -97,6 +91,10 @@ in {
zone influxdb 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "InfluxDB";
};
};
virtualHosts.${influxdbDomain} = let
accessRules = ''

15
hosts/sire/guests/loki.nix
View file

@ -18,11 +18,6 @@ in {
};
globals.services.loki.domain = lokiDomain;
globals.monitoring.http.loki = {
url = "https://${lokiDomain}";
location = "home";
network = "internet";
};
nodes.sentinel = {
age.secrets.loki-basic-auth-hashes = {
@ -38,6 +33,11 @@ in {
zone loki 64k;
keepalive 2;
'';
monitoring = {
enable = true;
path = "/ready";
expectedBodyRegex = "^ready";
};
};
virtualHosts.${lokiDomain} = {
forceSSL = true;
@ -80,6 +80,11 @@ in {
zone loki 64k;
keepalive 2;
'';
monitoring = {
enable = true;
path = "/ready";
expectedBodyRegex = "^ready";
};
};
virtualHosts.${lokiDomain} = {
forceSSL = true;

10
hosts/sire/guests/minecraft.nix
View file

@ -364,7 +364,11 @@ in {
globals.monitoring.tcp.minecraft = {
host = minecraftDomain;
port = 25565;
location = "home";
network = "internet";
};
globals.monitoring.http.minecraft-map = {
url = "https://${minecraftDomain}";
expectedBodyRegex = "Minecraft Dynamic Map";
network = "internet";
};
@ -400,6 +404,10 @@ in {
zone minecraft 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Minecraft Dynamic Map";
};
};
virtualHosts.${minecraftDomain} = {
forceSSL = true;

10
hosts/sire/guests/paperless.nix
View file

@ -27,7 +27,7 @@ in {
globals.services.paperless.domain = paperlessDomain;
globals.monitoring.http.paperless = {
url = "https://${paperlessDomain}";
location = "home";
expectedBodyRegex = "Paperless-ngx";
network = "internet";
};
@ -39,6 +39,10 @@ in {
zone paperless 64k;
keepalive 2;
'';
# direct upstream monitoring doesn't work because
# paperless allowed hosts fails for ip-based queries.
# But that's fine, we just monitor it via the domain above anyway.
#monitoring.enable = true;
};
virtualHosts.${paperlessDomain} = {
forceSSL = true;
@ -63,6 +67,10 @@ in {
zone paperless 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Paperless-ngx";
};
};
virtualHosts.${paperlessDomain} = {
forceSSL = true;

1
hosts/sire/guests/samba.nix
View file

@ -136,7 +136,6 @@ in {
globals.monitoring.tcp.samba = {
host = globals.net.home-lan.hosts.sire-samba.ipv4;
port = 445;
location = "home";
network = "home-lan";
};

1
hosts/sire/net.nix
View file

@ -9,7 +9,6 @@
globals.monitoring.ping.sire = {
hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv4;
hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.sire.cidrv6;
location = "home";
network = "home-lan";
};

10
hosts/ward/guests/adguardhome.nix
View file

@ -16,14 +16,8 @@ in {
globals.monitoring.dns.adguardhome = {
server = globals.net.home-lan.hosts.ward-adguardhome.ipv4;
domain = ".";
location = "home";
network = "home-lan";
};
globals.monitoring.http.adguardhome = {
url = "https://${adguardhomeDomain}";
location = "home";
network = "internet";
};
nodes.sentinel = {
services.nginx = {
@ -33,6 +27,10 @@ in {
zone adguardhome 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "AdGuard Home";
};
};
virtualHosts.${adguardhomeDomain} = {
forceSSL = true;

9
hosts/ward/guests/forgejo.nix
View file

@ -24,8 +24,8 @@ in {
globals.services.forgejo.domain = forgejoDomain;
globals.monitoring.http.forgejo = {
url = "https://${forgejoDomain}";
location = "home";
url = "https://${forgejoDomain}/user/login";
expectedBodyRegex = "Redlew Git";
network = "internet";
};
@ -57,6 +57,11 @@ in {
zone forgejo 64k;
keepalive 2;
'';
monitoring = {
enable = true;
path = "/user/login";
expectedBodyRegex = "Redlew Git";
};
};
virtualHosts.${forgejoDomain} = {
forceSSL = true;

12
hosts/ward/guests/kanidm.nix
View file

@ -41,9 +41,10 @@ in {
globals.services.kanidm.domain = kanidmDomain;
globals.monitoring.http.kanidm = {
url = "https://${kanidmDomain}";
location = "home";
url = "https://${kanidmDomain}/status";
network = "internet";
expectedBodyRegex = "true";
skipTlsVerification = true;
};
nodes.sentinel = {
@ -54,6 +55,13 @@ in {
zone kanidm 64k;
keepalive 2;
'';
monitoring = {
enable = true;
path = "/status";
expectedBodyRegex = "true";
skipTlsVerification = true;
useHttps = true;
};
};
virtualHosts.${kanidmDomain} = {
forceSSL = true;

11
hosts/ward/guests/netbird.nix
View file

@ -79,8 +79,9 @@ in {
globals.services.netbird.domain = netbirdDomain;
globals.monitoring.http.netbird = {
url = "https://${netbirdDomain}";
location = "home";
url = "https://${netbirdDomain}/api/users";
expectedStatus = 401;
expectedBodyRegex = "no valid authentication";
network = "internet";
};
@ -92,6 +93,12 @@ in {
zone netbird 64k;
keepalive 5;
'';
monitoring = {
enable = true;
path = "/api/users";
expectedStatus = 401;
expectedBodyRegex = "no valid authentication";
};
};
upstreams.netbird-signal = {

6
hosts/ward/guests/radicale.nix
View file

@ -9,7 +9,7 @@ in {
globals.services.radicale.domain = radicaleDomain;
globals.monitoring.http.radicale = {
url = "https://${radicaleDomain}";
location = "home";
expectedBodyRegex = "Radicale Web Interface";
network = "internet";
};
@ -21,6 +21,10 @@ in {
zone radicale 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Radicale Web Interface";
};
};
virtualHosts.${radicaleDomain} = {
forceSSL = true;

7
hosts/ward/guests/vaultwarden.nix
View file

@ -28,8 +28,7 @@ in {
globals.services.vaultwarden.domain = vaultwardenDomain;
globals.monitoring.http.vaultwarden = {
url = "https://${vaultwardenDomain}";
expectedBodyRegex = "Vaultwarden";
location = "home";
expectedBodyRegex = "Vaultwarden Web";
network = "internet";
};
@ -41,6 +40,10 @@ in {
zone vaultwarden 64k;
keepalive 2;
'';
monitoring = {
enable = true;
expectedBodyRegex = "Vaultwarden Web";
};
};
virtualHosts.${vaultwardenDomain} = {
forceSSL = true;

3
hosts/ward/guests/web-proxy.nix
View file

@ -11,6 +11,9 @@ in {
firewallRuleForAll.allowedTCPPorts = [80 443];
};
# This node shall monitor the infrastructure
meta.telegraf.availableMonitoringNetworks = ["internet" "home-wan" "home-lan"];
age.secrets.acme-cloudflare-dns-token = {
rekeyFile = config.node.secretsDir + "/acme-cloudflare-dns-token.age";
mode = "440";

1
hosts/ward/net.nix
View file

@ -9,7 +9,6 @@
globals.monitoring.ping.ward = {
hostv4 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv4;
hostv6 = lib.net.cidr.ip globals.net.home-lan.hosts.ward.cidrv6;
location = "home";
network = "home-lan";
};

1
hosts/zackbiene/net.nix
View file

@ -12,7 +12,6 @@ in {
globals.monitoring.ping.zackbiene = {
hostv4 = "zackbiene.local";
hostv6 = "zackbiene.local";
location = "home";
network = "home-lan";
};

1
modules/default.nix
View file

@ -13,6 +13,7 @@
./kanidm.nix
./meta.nix
./netbird-client.nix
./nginx-upstream-monitoring.nix
./oauth2-proxy.nix
./promtail.nix
./secrets.nix

157
modules/globals.nix
View file

@ -8,6 +8,13 @@
mkOption
types
;
defaultOptions = {
network = mkOption {
type = types.str;
description = "The network to which this endpoint is associated.";
};
};
in {
options = {
globals = mkOption {
@ -97,120 +104,94 @@ in {
monitoring = {
ping = mkOption {
type = types.attrsOf (types.submodule {
options = {
hostv4 = mkOption {
type = types.nullOr types.str;
description = "The IP/hostname to ping via ipv4.";
default = null;
};
options =
defaultOptions
// {
hostv4 = mkOption {
type = types.nullOr types.str;
description = "The IP/hostname to ping via ipv4.";
default = null;
};
hostv6 = mkOption {
type = types.nullOr types.str;
description = "The IP/hostname to ping via ipv6.";
default = null;
hostv6 = mkOption {
type = types.nullOr types.str;
description = "The IP/hostname to ping via ipv6.";
default = null;
};
};
location = mkOption {
type = types.str;
description = "A location tag added to this metric.";
};
network = mkOption {
type = types.str;
description = "The network to which this endpoint is associated.";
};
};
});
};
http = mkOption {
type = types.attrsOf (types.submodule {
options = {
url = mkOption {
type = types.str;
description = "The url to connect to.";
};
options =
defaultOptions
// {
url = mkOption {
type = types.either (types.listOf types.str) types.str;
description = "The url to connect to.";
};
location = mkOption {
type = types.str;
description = "A location tag added to this metric.";
};
expectedStatus = mkOption {
type = types.int;
default = 200;
description = "The HTTP status code to expect.";
};
network = mkOption {
type = types.str;
description = "The network to which this endpoint is associated.";
};
expectedBodyRegex = mkOption {
type = types.nullOr types.str;
description = "A regex pattern to expect in the body.";
default = null;
};
expectedStatus = mkOption {
type = types.int;
default = 200;
description = "The HTTP status code to expect.";
skipTlsVerification = mkOption {
type = types.bool;
description = "Skip tls verification when using https.";
default = false;
};
};
expectedBodyRegex = mkOption {
type = types.nullOr types.str;
description = "A regex pattern to expect in the body.";
default = null;
};
};
});
};
dns = mkOption {
type = types.attrsOf (types.submodule {
options = {
server = mkOption {
type = types.str;
description = "The DNS server to query.";
};
options =
defaultOptions
// {
server = mkOption {
type = types.str;
description = "The DNS server to query.";
};
domain = mkOption {
type = types.str;
description = "The domain to query.";
};
domain = mkOption {
type = types.str;
description = "The domain to query.";
};
record-type = mkOption {
type = types.str;
description = "The record type to query.";
default = "A";
record-type = mkOption {
type = types.str;
description = "The record type to query.";
default = "A";
};
};
location = mkOption {
type = types.str;
description = "A location tag added to this metric.";
};
network = mkOption {
type = types.str;
description = "The network to which this endpoint is associated.";
};
};
});
};
tcp = mkOption {
type = types.attrsOf (types.submodule {
options = {
host = mkOption {
type = types.str;
description = "The IP/hostname to connect to.";
};
options =
defaultOptions
// {
host = mkOption {
type = types.str;
description = "The IP/hostname to connect to.";
};
port = mkOption {
type = types.port;
description = "The port to connect to.";
port = mkOption {
type = types.port;
description = "The port to connect to.";
};
};
location = mkOption {
type = types.str;
description = "A location tag added to this metric.";
};
network = mkOption {
type = types.str;
description = "The network to which this endpoint is associated.";
};
};
});
};
};

81
modules/nginx-upstream-monitoring.nix Normal file
View file

@ -0,0 +1,81 @@
{
config,
lib,
...
}: let
inherit
(lib)
attrNames
filterAttrs
flip
mapAttrs'
mkOption
nameValuePair
types
;
in {
options.services.nginx.upstreams = mkOption {
type = types.attrsOf (types.submodule {
options.monitoring = {
enable = mkOption {
type = types.bool;
description = "Whether to add a global monitoring entry for this upstream";
default = false;
};
path = mkOption {
type = types.str;
description = "The path to query.";
default = "";
};
expectedStatus = mkOption {
type = types.int;
default = 200;
description = "The HTTP status code to expect.";
};
expectedBodyRegex = mkOption {
type = types.nullOr types.str;
description = "A regex pattern to expect in the body.";
default = null;
};
useHttps = mkOption {
type = types.bool;
description = "Whether to use https to connect to this upstream when monitoring";
default = false;
};
skipTlsVerification = mkOption {
type = types.bool;
description = "Skip tls verification when using https.";
default = false;
};
};
});
};
config = let
monitoredUpstreams = filterAttrs (_: x: x.monitoring.enable) config.services.nginx.upstreams;
in {
globals.monitoring.http = flip mapAttrs' monitoredUpstreams (
upstreamName: upstream: let
schema =
if upstream.monitoring.useHttps
then "https"
else "http";
in
nameValuePair "${config.node.name}-upstream-${upstreamName}" {
url = map (server: "${schema}://${server}${upstream.monitoring.path}") (attrNames upstream.servers);
network = "local-${config.node.name}";
inherit
(upstream.monitoring)
expectedBodyRegex
expectedStatus
skipTlsVerification
;
}
);
};
}

159
modules/telegraf.nix
View file

@ -21,6 +21,7 @@
optional
optionalAttrs
optionals
toList
types
;
@ -44,16 +45,14 @@ in {
description = "Additional secrets to replace in pre-start. The attr name will be searched and replaced in the config with the value read from the given file.";
};
globalMonitoring = {
enable = mkEnableOption "monitor the global infrastructure from this node.";
availableNetworks = mkOption {
type = types.listOf types.str;
example = ["internet"];
description = ''
The networks that can be reached from this node.
Only global entries with a matching network will be monitored from here.
'';
};
availableMonitoringNetworks = mkOption {
type = types.listOf types.str;
example = ["internet"];
description = ''
Any of the global monitoring definitions which has a network from this list
will automatically be monitored via telegraf. Set this to any networks that
can be reached from this node. This includes `local-<node.name>` by default.
'';
};
influxdb2 = {
@ -87,6 +86,9 @@ in {
};
config = mkIf (!minimal && cfg.enable) {
# Monitor anything that can only be monitored from this node
meta.telegraf.availableMonitoringNetworks = ["local-${config.node.name}"];
assertions = [
{
assertion = !config.boot.isContainer;
@ -185,6 +187,75 @@ in {
};
temp = {};
wireguard = {};
ping = concatLists (flip mapAttrsToList globals.monitoring.ping (
name: pingCfg:
optionals (elem pingCfg.network cfg.availableMonitoringNetworks) (
concatLists (forEach ["hostv4" "hostv6"] (
attr:
optional (pingCfg.${attr} != null) {
method = "native";
urls = [pingCfg.${attr}];
ipv4 = attr == "hostv4";
ipv6 = attr == "hostv6";
tags = {
inherit name;
inherit (pingCfg) network;
ip_version =
if attr == "hostv4"
then "v4"
else "v6";
};
fieldinclude = [
"percent_packet_loss"
"average_response_ms"
];
}
))
)
));
http_response = concatLists (flip mapAttrsToList globals.monitoring.http (
name: httpCfg:
optional (elem httpCfg.network cfg.availableMonitoringNetworks) {
urls = toList httpCfg.url;
method = "GET";
response_status_code = httpCfg.expectedStatus;
response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex;
insecure_skip_verify = httpCfg.skipTlsVerification;
follow_redirects = true;
tags = {
inherit name;
inherit (httpCfg) network;
};
}
));
dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns (
name: dnsCfg:
optional (elem dnsCfg.network cfg.availableMonitoringNetworks) {
servers = [dnsCfg.server];
domains = [dnsCfg.domain];
record_type = dnsCfg.record-type;
tags = {
inherit name;
inherit (dnsCfg) network;
};
}
));
net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp (
name: tcpCfg:
optional (elem tcpCfg.network cfg.availableMonitoringNetworks) {
address = "${tcpCfg.host}:${toString tcpCfg.port}";
protocol = "tcp";
tags = {
inherit name;
inherit (tcpCfg) network;
};
fieldexclude = ["result_type" "string_found"];
}
));
}
// optionalAttrs config.services.smartd.enable {
sensors = {};
@ -200,74 +271,6 @@ in {
}
// optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) {
wireless = {};
}
// optionalAttrs cfg.globalMonitoring.enable {
ping = concatLists (flip mapAttrsToList globals.monitoring.ping (
name: pingCfg:
optionals (elem pingCfg.network cfg.globalMonitoring.availableNetworks) (
concatLists (forEach ["hostv4" "hostv6"] (
attr:
optional (pingCfg.${attr} != null) {
method = "native";
urls = [pingCfg.${attr}];
ipv4 = attr == "hostv4";
ipv6 = attr == "hostv6";
tags = {
inherit name;
inherit (pingCfg) location network;
ip_version =
if attr == "hostv4"
then "v4"
else "v6";
};
fieldpass = [
"percent_packet_loss"
"average_response_ms"
];
}
))
)
));
http_response = concatLists (flip mapAttrsToList globals.monitoring.http (
name: httpCfg:
optional (elem httpCfg.network cfg.globalMonitoring.availableNetworks) {
urls = [httpCfg.url];
method = "GET";
response_status_code = httpCfg.expectedStatus;
response_string_match = mkIf (httpCfg.expectedBodyRegex != null) httpCfg.expectedBodyRegex;
tags = {
inherit name;
inherit (httpCfg) location network;
};
}
));
dns_query = concatLists (flip mapAttrsToList globals.monitoring.dns (
name: dnsCfg:
optional (elem dnsCfg.network cfg.globalMonitoring.availableNetworks) {
servers = [dnsCfg.server];
domains = [dnsCfg.domain];
record_type = dnsCfg.record-type;
tags = {
inherit name;
inherit (dnsCfg) location network;
};
}
));
net_response = concatLists (flip mapAttrsToList globals.monitoring.tcp (
name: tcpCfg:
optional (elem tcpCfg.network cfg.globalMonitoring.availableNetworks) {
address = "${tcpCfg.host}:${toString tcpCfg.port}";
protocol = "tcp";
tags = {
inherit name;
inherit (tcpCfg) location network;
};
fieldexclude = ["result_type" "string_found"];
}
));
};
};
};