diff --git a/hosts/nom/default.nix b/hosts/nom/default.nix index cfea34f..ee8229f 100644 --- a/hosts/nom/default.nix +++ b/hosts/nom/default.nix @@ -32,113 +32,4 @@ font = "ter-v28n"; packages = [pkgs.terminus_font]; }; - - services.influxdb2 = { - enable = true; - settings = { - reporting-disabled = true; - http-bind-address = "localhost:8086"; - }; - initialSetup = { - enable = true; - organization = "servers"; - bucket = "telegraf"; - - passwordFile = pkgs.writeText "tmp-pw" "ExAmPl3PA55W0rD"; - tokenFile = pkgs.writeText "tmp-tok" "asroiuhoiuahnawo4unhasdorviuhngoiuhraoug"; - }; - deleteOrganizations = ["delorg"]; - deleteBuckets = [ - { - name = "delbucket"; - org = "delorg"; - } - ]; - deleteUsers = ["deluser"]; - deleteRemotes = [ - { - name = "delremote"; - org = "delorg"; - } - ]; - deleteReplications = [ - { - name = "delreplication"; - org = "delorg"; - } - ]; - deleteApiTokens = [ - { - name = "deltoken"; - org = "delorg"; - user = "deluser"; - } - ]; - ensureOrganizations = [ - { - name = "myorg"; - description = "Myorg description"; - } - #{ - # name = "delorg"; - #} - ]; - ensureBuckets = [ - { - name = "mybucket"; - org = "myorg"; - description = "Mybucket description"; - } - #{ - # name = "delbucket"; - # org = "delorg"; - #} - ]; - ensureUsers = [ - { - name = "myuser"; - org = "myorg"; - passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga"; - } - #{ - # name = "deluser"; - # org = "delorg"; - # passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga"; - #} - ]; - #ensureRemotes = [ - # { - # name = "delremote"; - # org = "delorg"; - # remoteUrl = "http://localhost:8087"; - # remoteOrgId = "a1b2c3d4a1b2c3d4"; - # remoteTokenFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga"; - # } - #]; - #ensureReplications = [ - # { - # name = "delreplication"; - # org = "delorg"; - # remote = "delremote"; - # localBucket = "delbucket"; - # remoteBucket = "delbucket2"; - # } - #]; - ensureApiTokens = [ - { - name = "mytoken"; - org = "myorg"; - user = "myuser"; - readBuckets = ["mybucket"]; - writeBuckets = ["mybucket"]; - } - #{ - # name = "deltoken"; - # org = "delorg"; - # user = "deluser"; - # readBuckets = ["delbucket"]; - # writeBuckets = ["delbucket"]; - #} - ]; - }; } diff --git a/hosts/sentinel/default.nix b/hosts/sentinel/default.nix index d7c107f..2b0b7c5 100644 --- a/hosts/sentinel/default.nix +++ b/hosts/sentinel/default.nix @@ -29,8 +29,12 @@ networking.hosts.${config.meta.wireguard.proxy-sentinel.ipv4} = [config.networking.providedDomains.influxdb]; meta.telegraf = { enable = true; - influxdb2.domain = config.networking.providedDomains.influxdb; - influxdb2.organization = "servers"; - influxdb2.bucket = "telegraf"; + scrapeSensors = false; + influxdb2 = { + domain = config.networking.providedDomains.influxdb; + organization = "servers"; + bucket = "telegraf"; + node = "ward-influxdb"; + }; }; } diff --git a/hosts/ward/default.nix b/hosts/ward/default.nix index ca4c016..e74f3c4 100644 --- a/hosts/ward/default.nix +++ b/hosts/ward/default.nix @@ -32,9 +32,12 @@ networking.hosts.${nodes.sentinel.config.meta.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.networking.providedDomains.influxdb]; meta.telegraf = { enable = true; - influxdb2.domain = nodes.sentinel.config.networking.providedDomains.influxdb; - influxdb2.organization = "servers"; - influxdb2.bucket = "telegraf"; + influxdb2 = { + domain = nodes.sentinel.config.networking.providedDomains.influxdb; + organization = "servers"; + bucket = "telegraf"; + node = "ward-influxdb"; + }; }; # TODO track my github stats diff --git a/hosts/ward/microvms/common.nix b/hosts/ward/microvms/common.nix index 6e61ffd..33a4047 100644 --- a/hosts/ward/microvms/common.nix +++ b/hosts/ward/microvms/common.nix @@ -11,8 +11,12 @@ in { networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb]; meta.telegraf = { enable = true; - influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb; - influxdb2.organization = "servers"; - influxdb2.bucket = "telegraf"; + scrapeSensors = false; + influxdb2 = { + domain = sentinelCfg.networking.providedDomains.influxdb; + organization = "servers"; + bucket = "telegraf"; + node = "ward-influxdb"; + }; }; } diff --git a/hosts/ward/microvms/influxdb.nix b/hosts/ward/microvms/influxdb.nix index 5417c15..d44dd5c 100644 --- a/hosts/ward/microvms/influxdb.nix +++ b/hosts/ward/microvms/influxdb.nix @@ -3,6 +3,7 @@ lib, nodes, utils, + pkgs, ... }: let sentinelCfg = nodes.sentinel.config; @@ -51,14 +52,54 @@ in { }; }; + age.secrets.influxdb-admin-password = { + generator.script = "alnum"; + mode = "440"; + group = "influxdb2"; + }; + + age.secrets.influxdb-admin-token = { + generator.script = "alnum"; + mode = "440"; + group = "influxdb2"; + }; + + age.secrets.influxdb-user-telegraf-token = { + generator.script = "alnum"; + mode = "440"; + group = "influxdb2"; + }; + services.influxdb2 = { enable = true; settings = { reporting-disabled = true; http-bind-address = "${config.meta.wireguard.proxy-sentinel.ipv4}:${toString influxdbPort}"; }; + provision = { + enable = true; + initialSetup = { + organization = "default"; + bucket = "default"; + passwordFile = config.age.secrets.influxdb-admin-password.path; + tokenFile = config.age.secrets.influxdb-admin-token.path; + }; + ensureOrganizations = [ + { + name = "servers"; + } + ]; + ensureBuckets = [ + { + name = "telegraf"; + org = "servers"; + } + ]; + }; }; + environment.systemPackages = [pkgs.influxdb2-cli]; + systemd.services.influxdb2 = { after = ["sys-subsystem-net-devices-${utils.escapeSystemdPath "proxy-sentinel"}.device"]; serviceConfig.RestartSec = "600"; # Retry every 10 minutes diff --git a/hosts/zackbiene/default.nix b/hosts/zackbiene/default.nix index f60d215..0b5db61 100644 --- a/hosts/zackbiene/default.nix +++ b/hosts/zackbiene/default.nix @@ -47,9 +47,12 @@ in { networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb]; meta.telegraf = { enable = true; - influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb; - influxdb2.organization = "servers"; - influxdb2.bucket = "telegraf"; + influxdb2 = { + domain = sentinelCfg.networking.providedDomains.influxdb; + organization = "servers"; + bucket = "telegraf"; + node = "ward-influxdb"; + }; }; # Fails if there are no SMART devices diff --git a/modules/default.nix b/modules/default.nix index 5570d2d..2dd5dfb 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -18,6 +18,7 @@ ./config/users.nix ./config/xdg.nix + ./meta/influxdb-retrieve.nix ./meta/influxdb.nix ./meta/microvms.nix ./meta/nginx.nix diff --git a/modules/meta/influxdb-retrieve.nix b/modules/meta/influxdb-retrieve.nix new file mode 100644 index 0000000..03d77c4 --- /dev/null +++ b/modules/meta/influxdb-retrieve.nix @@ -0,0 +1,33 @@ +{ + config, + lib, + ... +}: let + inherit + (lib) + mkOption + types + ; + + cfg = config.services.influxdb2; +in { + options.services.influxdb2.provision.retrieveToken = mkOption { + type = types.functionTo (types.functionTo types.str); + readOnly = true; + description = "Script that returns a agenix-rekey generator to retrieve the given token"; + default = def: let + id = builtins.substring 0 32 (builtins.hashString "sha256" "${def.user}:${def.org}:${def.name}"); + in + { + pkgs, + lib, + ... + }: '' + echo " -> Retrieving influxdb token ${def.name} for org ${def.org} on ${config.node.name}" >&2 + ssh ${config.node.name} -- \ + 'bash -c '"'"'influx auth list --json --token "$(< ${cfg.provision.initialSetup.tokenFile})"'"'" \ + | ${lib.getExe pkgs.jq} -r '.[] | select(.description | contains("${id}")) | .token' \ + || die "Could not list/find influxdb api token '${def.name}' (${id})" + ''; + }; +} diff --git a/modules/meta/influxdb.nix b/modules/meta/influxdb.nix index 972f8c0..37edc84 100644 --- a/modules/meta/influxdb.nix +++ b/modules/meta/influxdb.nix @@ -31,9 +31,9 @@ cfg = config.services.influxdb2; in { - options.services.influxdb2 = { + options.services.influxdb2.provision = { + enable = mkEnableOption "initial database setup"; initialSetup = { - enable = mkEnableOption "initial database setup"; organization = mkOption { type = types.str; example = "main"; @@ -408,7 +408,7 @@ in { }; }; - config = mkIf (cfg.enable && cfg.initialSetup.enable) { + config = mkIf (cfg.enable && cfg.provision.enable) { assertions = let validPermissions = flip genAttrs (x: true) [ "authorizations" @@ -435,23 +435,23 @@ in { "replications" ]; - knownOrgs = map (x: x.name) cfg.ensureOrganizations; - knownRemotes = map (x: x.name) cfg.ensureRemotes; - knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.ensureBuckets); + knownOrgs = map (x: x.name) cfg.provision.ensureOrganizations; + knownRemotes = map (x: x.name) cfg.provision.ensureRemotes; + knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.provision.ensureBuckets); in - flip concatMap cfg.ensureBuckets (bucket: [ + flip concatMap cfg.provision.ensureBuckets (bucket: [ { assertion = elem bucket.org knownOrgs; message = "The influxdb bucket '${bucket.name}' refers to an unknown organization '${bucket.org}'."; } ]) - ++ flip concatMap cfg.ensureUsers (user: [ + ++ flip concatMap cfg.provision.ensureUsers (user: [ { assertion = elem user.org knownOrgs; message = "The influxdb user '${user.name}' refers to an unknown organization '${user.org}'."; } ]) - ++ flip concatMap cfg.ensureRemotes (remote: [ + ++ flip concatMap cfg.provision.ensureRemotes (remote: [ { assertion = (remote.remoteOrgId == null) != (remote.remoteOrg == null); message = "The influxdb remote '${remote.name}' must specify exactly one of remoteOrgId or remoteOrg."; @@ -461,19 +461,19 @@ in { message = "The influxdb remote '${remote.name}' refers to an unknown organization '${remote.org}'."; } ]) - ++ flip concatMap cfg.ensureReplications (replication: [ + ++ flip concatMap cfg.provision.ensureReplications (replication: [ { assertion = elem replication.remote knownRemotes; message = "The influxdb replication '${replication.name}' refers to an unknown remote '${replication.remote}'."; } (let - remote = head (filter (x: x.name == replication.remote) cfg.ensureRemotes); + remote = head (filter (x: x.name == replication.remote) cfg.provision.ensureRemotes); in { assertion = elem replication.localBucket (knownBucketsFor remote.org); message = "The influxdb replication '${replication.name}' refers to an unknown bucket '${replication.localBucket}' in organization '${remote.org}'."; }) ]) - ++ flip concatMap cfg.ensureApiTokens (apiToken: let + ++ flip concatMap cfg.provision.ensureApiTokens (apiToken: let validBuckets = flip genAttrs (x: true) (knownBucketsFor apiToken.org); in [ { @@ -557,20 +557,20 @@ in { # avoid saving the token as the active config. ${influxCli} setup \ --configs-path /dev/null \ - --org ${escapeShellArg cfg.initialSetup.organization} \ - --bucket ${escapeShellArg cfg.initialSetup.bucket} \ - --username ${escapeShellArg cfg.initialSetup.username} \ - --password "$(< ${escapeShellArg cfg.initialSetup.passwordFile})" \ - --token "$(< ${escapeShellArg cfg.initialSetup.tokenFile})" \ - --retention ${escapeShellArg cfg.initialSetup.retention} \ + --org ${escapeShellArg cfg.provision.initialSetup.organization} \ + --bucket ${escapeShellArg cfg.provision.initialSetup.bucket} \ + --username ${escapeShellArg cfg.provision.initialSetup.username} \ + --password "$(< ${escapeShellArg cfg.provision.initialSetup.passwordFile})" \ + --token "$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})" \ + --retention ${escapeShellArg cfg.provision.initialSetup.retention} \ --force >/dev/null rm -f "$STATE_DIRECTORY/.first_startup" fi - export INFLUX_TOKEN=$(< ${escapeShellArg cfg.initialSetup.tokenFile}) + export INFLUX_TOKEN=$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile}) '' - + flip concatMapStrings cfg.deleteApiTokens (apiToken: '' + + flip concatMapStrings cfg.provision.deleteApiTokens (apiToken: '' if id=$( ${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \ | ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id' @@ -579,7 +579,7 @@ in { echo "Deleted api token id="${escapeShellArg apiToken.id} fi '') - + flip concatMapStrings cfg.deleteReplications (replication: '' + + flip concatMapStrings cfg.provision.deleteReplications (replication: '' if id=$( ${influxCli} replication list --json --org ${escapeShellArg replication.org} --name ${escapeShellArg replication.name} 2>/dev/null \ | ${getExe pkgs.jq} -r ".[0].id" @@ -588,7 +588,7 @@ in { echo "Deleted replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name} fi '') - + flip concatMapStrings cfg.deleteRemotes (remote: '' + + flip concatMapStrings cfg.provision.deleteRemotes (remote: '' if id=$( ${influxCli} remote list --json --org ${escapeShellArg remote.org} --name ${escapeShellArg remote.name} 2>/dev/null \ | ${getExe pkgs.jq} -r ".[0].id" @@ -597,7 +597,7 @@ in { echo "Deleted remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name} fi '') - + flip concatMapStrings cfg.deleteUsers (user: '' + + flip concatMapStrings cfg.provision.deleteUsers (user: '' if id=$( ${influxCli} user list --json --name ${escapeShellArg user} 2>/dev/null \ | ${getExe pkgs.jq} -r ".[0].id" @@ -606,7 +606,7 @@ in { echo "Deleted user name="${escapeShellArg user} fi '') - + flip concatMapStrings cfg.deleteBuckets (bucket: '' + + flip concatMapStrings cfg.provision.deleteBuckets (bucket: '' if id=$( ${influxCli} bucket list --json --org ${escapeShellArg bucket.org} --name ${escapeShellArg bucket.name} 2>/dev/null \ | ${getExe pkgs.jq} -r ".[0].id" @@ -615,7 +615,7 @@ in { echo "Deleted bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name} fi '') - + flip concatMapStrings cfg.deleteOrganizations (org: '' + + flip concatMapStrings cfg.provision.deleteOrganizations (org: '' if id=$( ${influxCli} org list --json --name ${escapeShellArg org} 2>/dev/null \ | ${getExe pkgs.jq} -r ".[0].id" @@ -624,7 +624,7 @@ in { echo "Deleted org name="${escapeShellArg org} fi '') - + flip concatMapStrings cfg.ensureOrganizations (org: let + + flip concatMapStrings cfg.provision.ensureOrganizations (org: let listArgs = [ "--name" org.name @@ -645,7 +645,7 @@ in { echo "Created org name="${escapeShellArg org.name} fi '') - + flip concatMapStrings cfg.ensureBuckets (bucket: let + + flip concatMapStrings cfg.provision.ensureBuckets (bucket: let listArgs = [ "--org" bucket.org @@ -673,7 +673,7 @@ in { echo "Created bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name} fi '') - + flip concatMapStrings cfg.ensureUsers (user: let + + flip concatMapStrings cfg.provision.ensureUsers (user: let listArgs = [ "--name" user.name @@ -700,7 +700,7 @@ in { ${influxCli} user password ${escapeShellArgs listArgs} \ --password "$(< ${escapeShellArg user.passwordFile})" >/dev/null '') - + flip concatMapStrings cfg.ensureRemotes (remote: let + + flip concatMapStrings cfg.provision.ensureRemotes (remote: let listArgs = [ "--name" remote.name @@ -746,7 +746,7 @@ in { echo "Created remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name} fi '') - + flip concatMapStrings cfg.ensureReplications (replication: let + + flip concatMapStrings cfg.provision.ensureReplications (replication: let listArgs = [ "--name" replication.name @@ -780,7 +780,7 @@ in { echo "Created replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name} fi '') - + flip concatMapStrings cfg.ensureApiTokens (apiToken: let + + flip concatMapStrings cfg.provision.ensureApiTokens (apiToken: let listArgs = [ "--user" apiToken.user @@ -801,7 +801,7 @@ in { if id=$( ${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \ | ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id' - ); then + ) && [[ -n "$id" ]]; then true # No updateable args else declare -A bucketIds diff --git a/modules/meta/telegraf.nix b/modules/meta/telegraf.nix index 76a4f70..fc57ca6 100644 --- a/modules/meta/telegraf.nix +++ b/modules/meta/telegraf.nix @@ -7,7 +7,6 @@ }: let inherit (lib) - mdDoc mkEnableOption mkIf mkOption @@ -18,155 +17,179 @@ cfg = config.meta.telegraf; in { options.meta.telegraf = { - enable = mkEnableOption (mdDoc "telegraf to push metrics to influx."); + enable = mkEnableOption "telegraf to push metrics to influx."; + + scrapeSensors = mkOption { + type = types.bool; + default = true; + description = "Scrape sensors with lm_sensors. You should disable this for virtualized hosts."; + }; + influxdb2 = { domain = mkOption { type = types.str; example = "influxdb.example.com"; - description = mdDoc "The influxdb v2 database to push to. https will be enforced."; + description = "The influxdb v2 database to push to. https will be enforced."; }; organization = mkOption { type = types.str; - description = mdDoc "The organization to push to."; + description = "The organization to push to."; }; bucket = mkOption { type = types.str; - description = mdDoc "The bucket to push to."; + description = "The bucket to push to."; + }; + + user = mkOption { + type = types.str; + default = "admin"; + description = "The user for which the api key should be created."; + }; + + node = mkOption { + type = types.str; + description = "The node which hosts the influxdb service (used to provision an api token)."; }; }; }; - config = mkIf cfg.enable { - age.secrets.telegraf-influxdb-token = { - rekeyFile = config.node.secretsDir + "/telegraf-influxdb-token.age"; - # TODO generator.script = { pkgs, lib, decrypt, deps, ... }: let - # TODO adminBasicAuth = (builtins.head deps).file; - # TODO adminToken = (builtins.head deps).file; # TODO ..... filter by name? - # TODO in '' - # TODO echo " -> Provisioning influxdb token for telegraf on ${nodeName} at https://${cfg.influxdb2.domain}" >&2 - # TODO ${decrypt} ${lib.escapeShellArg aba.file} \ - # TODO INFLUX_HOST=https://${aba.host}+${aba.name}:${PW}@${URL} - # TODO | ${pkgs.influxdb2-cli}/bin/influx -niBC 12 ${lib.escapeShellArg host}"+"${lib.escapeShellArg name} \ - # TODO || die "Failure" - # TODO ''); - mode = "440"; - group = "telegraf"; + config = let + tokenDef = { + name = "telegraf (${config.node.name})"; + org = "servers"; + user = "admin"; + readBuckets = ["telegraf"]; + writeBuckets = ["telegraf"]; }; - - # It's intentional to gate this behind smartd. Hosts without smartd are usually - # virtual and won't need sensor collection either. - # XXX: maybe as separate option? - security.elewrap.telegraf-sensors = mkIf config.services.smartd.enable { - command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"]; - targetUser = "root"; - allowedUsers = ["telegraf"]; - }; - - security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable { - command = ["${pkgs.nvme-cli}/bin/nvme"]; - targetUser = "root"; - allowedUsers = ["telegraf"]; - passArguments = true; - }; - - security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable { - command = ["${pkgs.smartmontools}/bin/smartctl"]; - targetUser = "root"; - allowedUsers = ["telegraf"]; - passArguments = true; - }; - - services.telegraf = { - enable = true; - environmentFiles = [config.age.secrets.telegraf-influxdb-token.path]; - extraConfig = { - agent = { - interval = "10s"; - round_interval = true; # Always collect on :00,:10,... - metric_batch_size = 5000; - metric_buffer_limit = 50000; - collection_jitter = "0s"; - flush_interval = "20s"; - flush_jitter = "5s"; - precision = "1ms"; - hostname = config.node.name; - omit_hostname = false; - }; - outputs = { - influxdb_v2 = { - urls = ["https://${cfg.influxdb2.domain}"]; - token = "$INFLUX_TOKEN"; - inherit (cfg.influxdb2) organization bucket; - }; - }; - inputs = - { - conntrack = {}; - cpu = {}; - disk = {}; - diskio = {}; - internal = {}; - interrupts = {}; - kernel = {}; - kernel_vmstat = {}; - linux_sysctl_fs = {}; - mem = {}; - net = {}; - netstat = {}; - nstat = {}; - processes = {}; - swap = {}; - system = {}; - systemd_units = { - unittype = "service"; - }; - temp = {}; - wireguard = {}; - # http_response = { urls = [ "http://localhost/" ]; }; - # ping = { urls = [ "9.9.9.9" ]; }; - } - // optionalAttrs config.services.smartd.enable { - sensors = {}; - smart = { - attributes = true; - path_nvme = config.security.elewrap.telegraf-nvme.path; - path_smartctl = config.security.elewrap.telegraf-smartctl.path; - use_sudo = false; - }; - } - // optionalAttrs config.services.nginx.enable { - nginx.urls = ["http://localhost/nginx_status"]; - } - // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) { - wireless = {}; - }; - }; - }; - - services.nginx.virtualHosts = mkIf config.services.nginx.enable { - localhost.listenAddresses = ["127.0.0.1" "[::1]"]; - localhost.locations."= /nginx_status".extraConfig = '' - allow 127.0.0.0/8; - allow ::1; - deny all; - stub_status; - access_log off; - ''; - }; - - systemd.services.telegraf = { - path = [ - # Make sensors refer to the correct wrapper - (mkIf config.services.smartd.enable - (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path)) + in + mkIf cfg.enable { + nodes.${cfg.influxdb2.node}.services.influxdb2.provision.ensureApiTokens = [ + tokenDef ]; - serviceConfig = { - # For wireguard statistics - AmbientCapabilities = ["CAP_NET_ADMIN"]; - RestartSec = "600"; # Retry every 10 minutes + + age.secrets.telegraf-influxdb-token = { + generator = { + script = args: '' + echo -n "INFLUX_TOKEN=" + ${nodes.${cfg.influxdb2.node}.config.services.influxdb2.provision.retrieveToken tokenDef args} + ''; + tags = ["influxdb"]; + }; + mode = "440"; + group = "telegraf"; + }; + + security.elewrap.telegraf-sensors = mkIf cfg.scrapeSensors { + command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"]; + targetUser = "root"; + allowedUsers = ["telegraf"]; + }; + + security.elewrap.telegraf-nvme = mkIf config.services.smartd.enable { + command = ["${pkgs.nvme-cli}/bin/nvme"]; + targetUser = "root"; + allowedUsers = ["telegraf"]; + passArguments = true; + }; + + security.elewrap.telegraf-smartctl = mkIf config.services.smartd.enable { + command = ["${pkgs.smartmontools}/bin/smartctl"]; + targetUser = "root"; + allowedUsers = ["telegraf"]; + passArguments = true; + }; + + services.telegraf = { + enable = true; + environmentFiles = [config.age.secrets.telegraf-influxdb-token.path]; + extraConfig = { + agent = { + interval = "10s"; + round_interval = true; # Always collect on :00,:10,... + metric_batch_size = 5000; + metric_buffer_limit = 50000; + collection_jitter = "0s"; + flush_interval = "20s"; + flush_jitter = "5s"; + precision = "1ms"; + hostname = config.node.name; + omit_hostname = false; + }; + outputs = { + influxdb_v2 = { + urls = ["https://${cfg.influxdb2.domain}"]; + token = "$INFLUX_TOKEN"; + inherit (cfg.influxdb2) organization bucket; + }; + }; + inputs = + { + conntrack = {}; + cpu = {}; + disk = {}; + diskio = {}; + internal = {}; + interrupts = {}; + kernel = {}; + kernel_vmstat = {}; + linux_sysctl_fs = {}; + mem = {}; + net = {}; + netstat = {}; + nstat = {}; + processes = {}; + swap = {}; + system = {}; + systemd_units = { + unittype = "service"; + }; + temp = {}; + wireguard = {}; + # http_response = { urls = [ "http://localhost/" ]; }; + # ping = { urls = [ "9.9.9.9" ]; }; + } + // optionalAttrs config.services.smartd.enable { + sensors = {}; + smart = { + attributes = true; + path_nvme = config.security.elewrap.telegraf-nvme.path; + path_smartctl = config.security.elewrap.telegraf-smartctl.path; + use_sudo = false; + }; + } + // optionalAttrs config.services.nginx.enable { + nginx.urls = ["http://localhost/nginx_status"]; + } + // optionalAttrs (config.networking.wireless.enable || config.networking.wireless.iwd.enable) { + wireless = {}; + }; + }; + }; + + services.nginx.virtualHosts = mkIf config.services.nginx.enable { + localhost.listenAddresses = ["127.0.0.1" "[::1]"]; + localhost.locations."= /nginx_status".extraConfig = '' + allow 127.0.0.0/8; + allow ::1; + deny all; + stub_status; + access_log off; + ''; + }; + + systemd.services.telegraf = { + path = [ + # Make sensors refer to the correct wrapper + (mkIf cfg.scrapeSensors + (pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path)) + ]; + serviceConfig = { + # For wireguard statistics + AmbientCapabilities = ["CAP_NET_ADMIN"]; + RestartSec = "600"; # Retry every 10 minutes + }; }; }; - }; }