diff --git a/hosts/common/core/default.nix b/hosts/common/core/default.nix index cbaf96f..c04cf5d 100644 --- a/hosts/common/core/default.nix +++ b/hosts/common/core/default.nix @@ -21,6 +21,7 @@ ../../../modules/promtail.nix ../../../modules/proxied-domains.nix ../../../modules/repo.nix + ../../../modules/telegraf.nix ../../../modules/wireguard.nix ]; diff --git a/hosts/common/core/impermanence.nix b/hosts/common/core/impermanence.nix index 50d4ad5..7131d82 100644 --- a/hosts/common/core/impermanence.nix +++ b/hosts/common/core/impermanence.nix @@ -147,6 +147,14 @@ group = "influxdb2"; mode = "0700"; } + ] + ++ lib.optionals config.services.telegraf.enable [ + { + directory = "/var/lib/telegraf"; + user = "telegraf"; + group = "telegraf"; + mode = "0700"; + } ]; }; } diff --git a/hosts/common/core/system.nix b/hosts/common/core/system.nix index a4f4a6b..fa98a37 100644 --- a/hosts/common/core/system.nix +++ b/hosts/common/core/system.nix @@ -436,5 +436,6 @@ vaultwarden = uidGid 988; oauth2_proxy = uidGid 987; influxdb2 = uidGid 986; + telegraf = uidGid 985; }; } diff --git a/hosts/common/zfs.nix b/hosts/common/zfs.nix index 7f3df45..9a327a5 100644 --- a/hosts/common/zfs.nix +++ b/hosts/common/zfs.nix @@ -1,6 +1,7 @@ { - pkgs, config, + lib, + pkgs, ... }: { boot.supportedFilesystems = ["zfs"]; @@ -22,4 +23,8 @@ interval = "weekly"; }; }; + + services.telegraf.extraConfig.inputs = lib.mkIf config.services.telegraf.enable { + zfs.poolMetrics = true; + }; } diff --git a/hosts/ward/default.nix b/hosts/ward/default.nix index 2871410..169bdec 100644 --- a/hosts/ward/default.nix +++ b/hosts/ward/default.nix @@ -25,6 +25,16 @@ proxy = "sentinel"; }; + extra.telegraf = { + enable = true; + proxy = "sentinel"; + # TODO organization = "servers"; + # TODO bucket = "telegraf"; + }; + + # TODO track my github stats + # services.telegraf.extraConfig.inputs.github = {}; + extra.microvms.vms = let defaults = { system = "x86_64-linux"; @@ -53,9 +63,6 @@ #minecraft = defineVm; #firefly - #prometheus - #influxdb - #maddy = defineVm; #anonaddy = defineVm; diff --git a/hosts/ward/microvms/grafana/default.nix b/hosts/ward/microvms/grafana/default.nix index 1be7164..46ea921 100644 --- a/hosts/ward/microvms/grafana/default.nix +++ b/hosts/ward/microvms/grafana/default.nix @@ -35,9 +35,8 @@ in { group = "grafana"; }; - age.secrets.grafana-influxdb-basic-auth-password = { - rekeyFile = ./secrets/grafana-influxdb-basic-auth-password.age; - generator = "alnum"; + age.secrets.grafana-influxdb-token = { + rekeyFile = ./secrets/grafana-influxdb-token.age; mode = "440"; group = "grafana"; }; @@ -47,10 +46,6 @@ in { config.age.secrets.grafana-loki-basic-auth-password ]; - age.secrets.influxdb-basic-auth-hashes.generator.dependencies = [ - config.age.secrets.grafana-influxdb-basic-auth-password - ]; - proxiedDomains.grafana = grafanaDomain; services.nginx = { @@ -126,10 +121,7 @@ in { access = "proxy"; url = "https://${sentinelCfg.proxiedDomains.influxdb}"; orgId = 1; - basicAuth = true; - basicAuthUser = "${nodeName}+grafana-influxdb-basic-auth-password"; - secureJsonData.basicAuthPassword = "$__file{${config.age.secrets.grafana-influxdb-basic-auth-password.path}}"; - #secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}"; + secureJsonData.token = "$__file{${config.age.secrets.grafana-influxdb-token.path}}"; jsonData.version = "Flux"; } { diff --git a/hosts/ward/microvms/grafana/secrets/grafana-influxdb-token.age b/hosts/ward/microvms/grafana/secrets/grafana-influxdb-token.age new file mode 100644 index 0000000..66e5f02 --- /dev/null +++ b/hosts/ward/microvms/grafana/secrets/grafana-influxdb-token.age @@ -0,0 +1,10 @@ +age-encryption.org/v1 +-> X25519 R7qu+/ASvapcZPEO2cXqEisYoiwSims+RSuCRCZsYgw +lcYaTINCZY3g6DVCgejc3twlVdcMJEQE066ANZlYy8M +-> piv-p256 xqSe8Q A1Wtem3odHpc6a3PSFShHkm2L8MeLbtv2ofmCfisFWXp ++OFZvkDksOUSmqOQVPLL5Ytk8eBkEmSYv2jc9jNOJWw +-> @/*G-grease {vMz+#o gEzN;NY "+/{ +UP0 +SL0mOBt6VjU+vI+1fXKz52Ch0V7tgxX7k4emorUCC+GFOgo3BLrTu6R23Xhnoub3 +zw +--- kKnHHEZuf1z2jYZpo5Zbvffh5AVG+VztRauG1mFkUYw +Jk%ycq>@_l=V}a"_I҄x>.BnBC)~9AZgd Ul~ &QW6TGvƓ/'v+ \ No newline at end of file diff --git a/hosts/ward/microvms/influxdb/default.nix b/hosts/ward/microvms/influxdb/default.nix index 56724a2..2f91661 100644 --- a/hosts/ward/microvms/influxdb/default.nix +++ b/hosts/ward/microvms/influxdb/default.nix @@ -27,15 +27,6 @@ in { nodes.sentinel = { proxiedDomains.influxdb = influxdbDomain; - age.secrets.influxdb-basic-auth-hashes = { - rekeyFile = ./secrets/influxdb-basic-auth-hashes.age; - # Copy only the script so the dependencies can be added by the nodes - # that define passwords (using distributed-config). - generator.script = config.age.generators.basic-auth.script; - mode = "440"; - group = "nginx"; - }; - services.nginx = { upstreams.influxdb = { servers."${config.services.influxdb2.settings.http-bind-address}" = {}; @@ -54,9 +45,8 @@ in { proxyWebsockets = true; extraConfig = '' satisfy any; - auth_basic "Authentication required"; - auth_basic_user_file ${sentinelCfg.age.secrets.influxdb-basic-auth-hashes.path}; - access_log off; + ${lib.concatMapStrings (ip: "allow ${ip};\n") sentinelCfg.extra.wireguard.proxy-sentinel.server.reservedAddresses}; + deny all; ''; }; }; diff --git a/hosts/ward/secrets/telegraf-influxdb-token.age b/hosts/ward/secrets/telegraf-influxdb-token.age new file mode 100644 index 0000000..b2da0f5 --- /dev/null +++ b/hosts/ward/secrets/telegraf-influxdb-token.age @@ -0,0 +1,13 @@ +age-encryption.org/v1 +-> X25519 DCVhASEENA4z7QkZIAz+7shz69B3UGfuR4QwV28e3w4 +KcvcVb5PxsRMlA5n35c/4nRLdv7WoIL2bqJn6Ry0tBU +-> piv-p256 xqSe8Q ArDV5TYzLEFhnRxXIY1OMPe4nPE7rtNhsUhU+7J2La3o +SoqSbbPvxlF4uaGSRNKSumajM9aEr2EoHE8PyPr3sMk +-> e\9`z-grease +PtN7lO2jjyBoMojXSiPLmWGgv23uUbzd9TxrAwwDiCcBbW5RL5vvR2HFzc+k+ZVa +RA3xLg5UeIzjsZdkWBezPHX1p7OALN49ZxtJ21fzfDhdUCTfVIKK4mi++At2hEJF +6g +--- FdR7X/jFWv+BhzuO8kpGr8xC3SKgmrwHg4YaHRxnwHE + p 6SDW(YR3s_۪/2g}ܶW?ub +)/ ,{&F-5/u.p \%ɟ4qӉyKQk4W3ˌ +ӈ[t__4y q^/F* \ No newline at end of file diff --git a/hosts/zackbiene/default.nix b/hosts/zackbiene/default.nix index 9fb5c17..f987e31 100644 --- a/hosts/zackbiene/default.nix +++ b/hosts/zackbiene/default.nix @@ -8,8 +8,9 @@ imports = [ ../common/core ../common/hardware/odroid-n2plus.nix - #../common/initrd-ssh.nix + ../common/initrd-ssh.nix ../common/zfs.nix + ../common/bios-boot.nix ./fs.nix ./net.nix @@ -23,10 +24,7 @@ ./zigbee2mqtt.nix ]; - # TODO replace by bios-boot.nix - # and grub.devices = ... once disko is in use. - boot.loader.grub.enable = false; - boot.loader.generic-extlinux-compatible.enable = true; + # TODO boot.loader.grub.devices = ["/dev/disk/by-id/${config.repo.secrets.local.disk.main}"]; console.earlySetup = true; # Fails if there are no SMART devices diff --git a/hosts/zackbiene/hostapd.nix b/hosts/zackbiene/hostapd.nix index 0996dfa..787e0d8 100644 --- a/hosts/zackbiene/hostapd.nix +++ b/hosts/zackbiene/hostapd.nix @@ -10,6 +10,8 @@ # Associates each known client to a unique password age.secrets.wifi-clients.rekeyFile = ./secrets/wifi-clients.age; + hardware.wirelessRegulatoryDatabase = true; + services.hostapd = { enable = true; radios.wlan1 = { diff --git a/modules/extra.nix b/modules/extra.nix index b852c18..49b6a16 100644 --- a/modules/extra.nix +++ b/modules/extra.nix @@ -117,8 +117,20 @@ in { access_log syslog:server=unix:/dev/log,nohostname json_combined; ssl_ecdh_curve secp384r1; ''; + + virtualHosts.localhost = { + locations."= /nginx_status".extraConfig = '' + allow 127.0.0.0/8; + deny all; + stub_status; + ''; + }; }; networking.firewall.allowedTCPPorts = optionals config.services.nginx.enable [80 443]; + + services.telegraf.extraConfig.inputs = mkIf config.services.nginx.enable { + nginx.urls = ["http://localhost/nginx_status"]; + }; }; } diff --git a/modules/telegraf.nix b/modules/telegraf.nix new file mode 100644 index 0000000..fbebaa7 --- /dev/null +++ b/modules/telegraf.nix @@ -0,0 +1,110 @@ +{ + config, + lib, + nodeName, + nodePath, + nodes, + pkgs, + ... +}: let + inherit + (lib) + mdDoc + mkEnableOption + mkIf + mkOption + optionalAttrs + types + ; + + cfg = config.extra.telegraf; +in { + options.extra.telegraf = { + enable = mkEnableOption (mdDoc "telegraf to push metrics to influx."); + proxy = mkOption { + type = types.str; + description = mdDoc "The node name of the proxy server which provides the influx api endpoint."; + }; + }; + + config = mkIf cfg.enable { + # Connect safely via wireguard to skip authentication + networking.hosts.${nodes.${cfg.proxy}.config.extra.wireguard."proxy-${cfg.proxy}".ipv4} = [nodes.${cfg.proxy}.config.proxiedDomains.influxdb]; + + age.secrets.telegraf-influxdb-token = { + rekeyFile = nodePath + "/secrets/telegraf-influxdb-token.age"; + mode = "440"; + group = "telegraf"; + }; + + services.telegraf = { + enable = true; + environmentFiles = [config.age.secrets.telegraf-influxdb-token.path]; + extraConfig = { + agent = { + interval = "10s"; + round_interval = true; # Always collect on :00,:10,... + metric_batch_size = 5000; + metric_buffer_limit = 50000; + collection_jitter = "0s"; + flush_interval = "20s"; + flush_jitter = "5s"; + precision = "1ms"; + hostname = nodeName; + omit_hostname = false; + }; + outputs = { + influxdb_v2 = { + urls = ["https://${nodes.${cfg.proxy}.config.proxiedDomains.influxdb}"]; + token = "$INFLUX_TOKEN"; + organization = "servers"; + bucket = "telegraf"; + }; + }; + inputs = + { + conntrack = {}; + cpu = {}; + disk = {}; + diskio = {}; + internal = {}; + interrupts = {}; + kernel = {}; + kernel_vmstat = {}; + linux_sysctl_fs = {}; + mem = {}; + net = {}; + netstat = {}; + nstat = {}; + processes = {}; + sensors = {}; + swap = {}; + system = {}; + systemd_units = {unittype = "service";}; + temp = {}; + wireguard = {}; + # http_response = { urls = [ "http://localhost/" ]; }; + # ping = { urls = [ "9.9.9.9" ]; }; + } + // optionalAttrs config.services.smartd.enable { + smart = { + path_nvme = "${pkgs.nvme-cli}/bin/nvme"; + path_smartctl = "${pkgs.smartmontools}/bin/smartctl"; + use_sudo = true; + }; + # TODO } // optionalAttrs config.services.iwd.enable { + # TODO wireless = { }; + }; + }; + }; + + systemd.services.telegraf = { + path = [ + "/run/wrappers" + pkgs.lm_sensors + ]; + # For wireguard statistics + serviceConfig.AmbientCapabilities = ["CAP_NET_ADMIN"]; + }; + }; +} diff --git a/modules/wireguard.nix b/modules/wireguard.nix index 08a2493..1bc31b1 100644 --- a/modules/wireguard.nix +++ b/modules/wireguard.nix @@ -137,7 +137,6 @@ assertion = stringLength wgCfg.linkName < 16; message = "${assertionPrefix}: The specified linkName '${wgCfg.linkName}' is too long (must be max 15 characters)."; } - # TODO at least 3 network participants and (externalPeers != {} or someone has via set to us) -> ip forwarding ]; networking.firewall.allowedUDPPorts =