feat: enable telegraf on all server nodes; add 10 minute autorestart

This commit is contained in:
oddlama 2023-06-25 02:34:05 +02:00
parent b8f647fb4a
commit f606e6e554
No known key found for this signature in database
GPG key ID: 14EFE510775FE39A
25 changed files with 228 additions and 41 deletions

View file

@ -19,7 +19,7 @@
../../../modules/microvms.nix
../../../modules/oauth2-proxy.nix
../../../modules/promtail.nix
../../../modules/proxied-domains.nix
../../../modules/provided-domains.nix
../../../modules/repo.nix
../../../modules/telegraf.nix
../../../modules/wireguard.nix

View file

@ -24,4 +24,13 @@
enable = true;
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${config.extra.wireguard.proxy-sentinel.ipv4} = [config.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = config.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
}

View file

@ -40,6 +40,15 @@
networking.nftables.firewall = {
zones = lib.mkForce {
untrusted.interfaces = ["wan"];
proxy-sentinel.interfaces = ["proxy-sentinel"];
};
rules = lib.mkForce {
# Allow accessing nginx through the proxy
proxy-sentinel-to-local = {
from = ["proxy-sentinel"];
to = ["local"];
allowedTCPPorts = [80 443];
};
};
};

View file

@ -22,15 +22,15 @@
in {
provider = "oidc";
scope = "openid email";
loginURL = "https://${config.proxiedDomains.kanidm}/ui/oauth2";
redeemURL = "https://${config.proxiedDomains.kanidm}/oauth2/token";
validateURL = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
loginURL = "https://${config.providedDomains.kanidm}/ui/oauth2";
redeemURL = "https://${config.providedDomains.kanidm}/oauth2/token";
validateURL = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}/userinfo";
clientID = clientId;
keyFile = config.age.secrets.oauth2-proxy-secret.path;
email.domains = ["*"];
extraConfig = {
oidc-issuer-url = "https://${config.proxiedDomains.kanidm}/oauth2/openid/${clientId}";
oidc-issuer-url = "https://${config.providedDomains.kanidm}/oauth2/openid/${clientId}";
provider-display-name = "Kanidm";
#skip-provider-button = true;
};

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -1,6 +1,7 @@
{
config,
nixos-hardware,
nodes,
...
}: {
imports = [
@ -25,11 +26,13 @@
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${nodes.sentinel.config.extra.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.providedDomains.influxdb];
extra.telegraf = {
enable = true;
proxy = "sentinel";
# TODO organization = "servers";
# TODO bucket = "telegraf";
influxdb2.url = nodes.sentinel.config.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
# TODO track my github stats

View file

@ -17,12 +17,21 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
networking.nftables.firewall.rules = lib.mkForce {
sentinel-to-local.allowedTCPPorts = [config.services.adguardhome.settings.bind_port];
};
nodes.sentinel = {
proxiedDomains.adguard = adguardhomeDomain;
providedDomains.adguard = adguardhomeDomain;
services.nginx = {
upstreams.adguardhome = {

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -18,6 +18,15 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
networking.nftables.firewall.rules = lib.mkForce {
sentinel-to-local.allowedTCPPorts = [config.services.grafana.settings.server.http_port];
};
@ -46,7 +55,7 @@ in {
config.age.secrets.grafana-loki-basic-auth-password
];
proxiedDomains.grafana = grafanaDomain;
providedDomains.grafana = grafanaDomain;
services.nginx = {
upstreams.grafana = {
@ -102,9 +111,9 @@ in {
client_secret = "aZKNCM6KpjBy4RqwKJXMLXzyx9rKH6MZTFk4wYrKWuBqLj6t"; # TODO temporary test not a real secret
scopes = "openid email profile";
login_attribute_path = "prefered_username";
auth_url = "https://${sentinelCfg.proxiedDomains.kanidm}/ui/oauth2";
token_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/token";
api_url = "https://${sentinelCfg.proxiedDomains.kanidm}/oauth2/openid/grafana/userinfo";
auth_url = "https://${sentinelCfg.providedDomains.kanidm}/ui/oauth2";
token_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/token";
api_url = "https://${sentinelCfg.providedDomains.kanidm}/oauth2/openid/grafana/userinfo";
use_pkce = true;
# Allow mapping oauth2 roles to server admin
allow_assign_grafana_admin = true;
@ -116,19 +125,22 @@ in {
enable = true;
datasources.settings.datasources = [
{
name = "InfluxDB";
name = "InfluxDB (servers)";
type = "influxdb";
access = "proxy";
url = "https://${sentinelCfg.proxiedDomains.influxdb}";
url = "https://${sentinelCfg.providedDomains.influxdb}";
orgId = 1;
secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}";
jsonData.version = "Flux";
jsonData.organization = "servers";
jsonData.defaultBucket = "telegraf";
}
# TODO duplicate above influxdb source (with scoped read tokens??) for each organization
{
name = "Loki";
type = "loki";
access = "proxy";
url = "https://${sentinelCfg.proxiedDomains.loki}";
url = "https://${sentinelCfg.providedDomains.loki}";
orgId = 1;
basicAuth = true;
basicAuthUser = "${nodeName}+grafana-loki-basic-auth-password";

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -20,12 +20,21 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
networking.nftables.firewall.rules = lib.mkForce {
sentinel-to-local.allowedTCPPorts = [influxdbPort];
};
nodes.sentinel = {
proxiedDomains.influxdb = influxdbDomain;
providedDomains.influxdb = influxdbDomain;
services.nginx = {
upstreams.influxdb = {
@ -45,7 +54,7 @@ in {
proxyWebsockets = true;
extraConfig = ''
satisfy any;
${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses};
${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses}
deny all;
'';
};

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -19,6 +19,15 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
networking.nftables.firewall.rules = lib.mkForce {
sentinel-to-local.allowedTCPPorts = [kanidmPort];
};
@ -36,7 +45,7 @@ in {
};
nodes.sentinel = {
proxiedDomains.kanidm = kanidmDomain;
providedDomains.kanidm = kanidmDomain;
services.nginx = {
upstreams.kanidm = {

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -17,12 +17,21 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
networking.nftables.firewall.rules = lib.mkForce {
sentinel-to-local.allowedTCPPorts = [config.services.loki.configuration.server.http_listen_port];
};
nodes.sentinel = {
proxiedDomains.loki = lokiDomain;
providedDomains.loki = lokiDomain;
age.secrets.loki-basic-auth-hashes = {
rekeyFile = ./secrets/loki-basic-auth-hashes.age;

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -17,6 +17,15 @@ in {
proxy = "sentinel";
};
# Connect safely via wireguard to skip authentication
networking.hosts.${sentinelCfg.extra.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.providedDomains.influxdb];
extra.telegraf = {
enable = true;
influxdb2.url = sentinelCfg.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
};
age.secrets.vaultwarden-env = {
rekeyFile = ./secrets/vaultwarden-env.age;
mode = "440";
@ -31,7 +40,7 @@ in {
};
nodes.sentinel = {
proxiedDomains.vaultwarden = vaultwardenDomain;
providedDomains.vaultwarden = vaultwardenDomain;
upstreams.vaultwarden = {
servers."${config.services.vaultwarden.config.rocketAddress}:${toString config.services.vaultwarden.config.rocketPort}" = {};

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4
KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU
-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o
SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk
-> e\9`z-grease
PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa
RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF
6g
--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE
¶ßÌÚÝp 6SD´…˜W(¶YRÜ3ƒs_Ûª/2g}Äܶ§W?ub
)‘¯/û ,{÷&ƒFÿ-ŒØ5£ß/u.�p¬ \%ÊÉŸæ—üï4qÓ‰�ðÛ˜yKQk4W™3÷ËŒ
§Óˆ[Áþ°t‡__4y× ±q�¬^/Fש*

View file

@ -177,4 +177,7 @@ in {
macvtapInterface = "lan";
wireguard.openFirewallRules = ["lan-to-local"];
};
# Allow accessing influx
extra.wireguard.proxy-sentinel.client.via = "sentinel";
}

View file

@ -37,7 +37,7 @@ in {
allNodes = attrNames colmenaNodes;
isColmenaNode = elem nodeName allNodes;
foreignConfigs = concatMap (n: colmenaNodes.${n}.config.nodes.${nodeName} or []) allNodes;
toplevelAttrs = ["age" "proxiedDomains" "networking" "systemd" "services"];
toplevelAttrs = ["age" "providedDomains" "networking" "systemd" "services"];
in
optionalAttrs isColmenaNode (mergeToplevelConfigs toplevelAttrs (
foreignConfigs

View file

@ -125,6 +125,7 @@ in {
RuntimeDirectory = "oauth2_proxy";
RuntimeDirectoryMode = "0750";
UMask = "007"; # TODO remove once https://github.com/oauth2-proxy/oauth2-proxy/issues/2141 is fixed
RestartSec = "600"; # Retry every 10 minutes
};
users.groups.oauth2_proxy.members = ["nginx"];

View file

@ -50,7 +50,7 @@ in {
{
basic_auth.username = "${nodeName}+promtail-loki-basic-auth-password";
basic_auth.password_file = config.age.secrets.promtail-loki-basic-auth-password.path;
url = "https://${nodes.${cfg.proxy}.config.proxiedDomains.loki}/loki/api/v1/push";
url = "https://${nodes.${cfg.proxy}.config.providedDomains.loki}/loki/api/v1/push";
}
];
@ -147,5 +147,7 @@ in {
];
};
};
systemd.services.promtail.serviceConfig.RestartSec = "600"; # Retry every 10 minutes
};
}

View file

@ -0,0 +1,7 @@
{lib, ...}: {
options.providedDomains = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = {};
description = "Registry of domains that this host 'provides' (that refer to this host with some functionality). For easy cross-node referencing.";
};
}

View file

@ -1,7 +0,0 @@
{lib, ...}: {
options.proxiedDomains = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = {};
description = "Registry of proxied domains for easy cross-node referencing.";
};
}

View file

@ -21,16 +21,26 @@
in {
options.extra.telegraf = {
enable = mkEnableOption (mdDoc "telegraf to push metrics to influx.");
proxy = mkOption {
type = types.str;
description = mdDoc "The node name of the proxy server which provides the influx api endpoint.";
influxdb2 = {
url = mkOption {
type = types.str;
example = "https://influxdb.example.com";
description = mdDoc "The influxdb v2 database url to push to.";
};
organization = mkOption {
type = types.str;
description = mdDoc "The organization to push to.";
};
bucket = mkOption {
type = types.str;
description = mdDoc "The bucket to push to.";
};
};
};
config = mkIf cfg.enable {
# Connect safely via wireguard to skip authentication
networking.hosts.${nodes.${cfg.proxy}.config.extra.wireguard."proxy-${cfg.proxy}".ipv4} = [nodes.${cfg.proxy}.config.proxiedDomains.influxdb];
age.secrets.telegraf-influxdb-token = {
rekeyFile = nodePath + "/secrets/telegraf-influxdb-token.age";
mode = "440";
@ -55,10 +65,9 @@ in {
};
outputs = {
influxdb_v2 = {
urls = ["https://${nodes.${cfg.proxy}.config.proxiedDomains.influxdb}"];
urls = [cfg.influxdb2.url];
token = "$INFLUX_TOKEN";
organization = "servers";
bucket = "telegraf";
inherit (cfg.influxdb2) organization bucket;
};
};
inputs =
@ -103,8 +112,11 @@ in {
"/run/wrappers"
pkgs.lm_sensors
];
# For wireguard statistics
serviceConfig.AmbientCapabilities = ["CAP_NET_ADMIN"];
serviceConfig = {
# For wireguard statistics
AmbientCapabilities = ["CAP_NET_ADMIN"];
RestartSec = "600"; # Retry every 10 minutes
};
};
};
}