mirrors_public/oddlama_nix-config
1
1
Fork 1
mirror of https://github.com/oddlama/nix-config.git synced 2025-10-11 15:20:39 +02:00
Code Activity

feat: automatically provision tokens for telegraf

Browse source
This commit is contained in:
oddlama 2023-08-16 00:17:47 +02:00
parent 6a14451033
commit 377da23c0d
No known key found for this signature in database
GPG key ID: 14EFE510775FE39A
10 changed files with 288 additions and 285 deletions
Download patch file Download diff file Expand all files Collapse all files

109
hosts/nom/default.nix
View file

@ -32,113 +32,4 @@
font = "ter-v28n";
packages = [pkgs.terminus_font];
};
services.influxdb2 = {
enable = true;
settings = {
reporting-disabled = true;
http-bind-address = "localhost:8086";
};
initialSetup = {
enable = true;
organization = "servers";
bucket = "telegraf";
passwordFile = pkgs.writeText "tmp-pw" "ExAmPl3PA55W0rD";
tokenFile = pkgs.writeText "tmp-tok" "asroiuhoiuahnawo4unhasdorviuhngoiuhraoug";
};
deleteOrganizations = ["delorg"];
deleteBuckets = [
{
name = "delbucket";
org = "delorg";
}
];
deleteUsers = ["deluser"];
deleteRemotes = [
{
name = "delremote";
org = "delorg";
}
];
deleteReplications = [
{
name = "delreplication";
org = "delorg";
}
];
deleteApiTokens = [
{
name = "deltoken";
org = "delorg";
user = "deluser";
}
];
ensureOrganizations = [
{
name = "myorg";
description = "Myorg description";
}
#{
# name = "delorg";
#}
];
ensureBuckets = [
{
name = "mybucket";
org = "myorg";
description = "Mybucket description";
}
#{
# name = "delbucket";
# org = "delorg";
#}
];
ensureUsers = [
{
name = "myuser";
org = "myorg";
passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
}
#{
# name = "deluser";
# org = "delorg";
# passwordFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
#}
];
#ensureRemotes = [
# {
# name = "delremote";
# org = "delorg";
# remoteUrl = "http://localhost:8087";
# remoteOrgId = "a1b2c3d4a1b2c3d4";
# remoteTokenFile = pkgs.writeText "tmp-pw" "abcgoiuhaoga";
# }
#];
#ensureReplications = [
# {
# name = "delreplication";
# org = "delorg";
# remote = "delremote";
# localBucket = "delbucket";
# remoteBucket = "delbucket2";
# }
#];
ensureApiTokens = [
{
name = "mytoken";
org = "myorg";
user = "myuser";
readBuckets = ["mybucket"];
writeBuckets = ["mybucket"];
}
#{
# name = "deltoken";
# org = "delorg";
# user = "deluser";
# readBuckets = ["delbucket"];
# writeBuckets = ["delbucket"];
#}
];
};
}

10
hosts/sentinel/default.nix
View file

@ -29,8 +29,12 @@
networking.hosts.${config.meta.wireguard.proxy-sentinel.ipv4} = [config.networking.providedDomains.influxdb];
meta.telegraf = {
enable = true;
influxdb2.domain = config.networking.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
scrapeSensors = false;
influxdb2 = {
domain = config.networking.providedDomains.influxdb;
organization = "servers";
bucket = "telegraf";
node = "ward-influxdb";
};
};
}

9
hosts/ward/default.nix
View file

@ -32,9 +32,12 @@
networking.hosts.${nodes.sentinel.config.meta.wireguard.proxy-sentinel.ipv4} = [nodes.sentinel.config.networking.providedDomains.influxdb];
meta.telegraf = {
enable = true;
influxdb2.domain = nodes.sentinel.config.networking.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
influxdb2 = {
domain = nodes.sentinel.config.networking.providedDomains.influxdb;
organization = "servers";
bucket = "telegraf";
node = "ward-influxdb";
};
};
# TODO track my github stats

10
hosts/ward/microvms/common.nix
View file

@ -11,8 +11,12 @@ in {
networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb];
meta.telegraf = {
enable = true;
influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
scrapeSensors = false;
influxdb2 = {
domain = sentinelCfg.networking.providedDomains.influxdb;
organization = "servers";
bucket = "telegraf";
node = "ward-influxdb";
};
};
}

41
hosts/ward/microvms/influxdb.nix
View file

@ -3,6 +3,7 @@
lib,
nodes,
utils,
pkgs,
...
}: let
sentinelCfg = nodes.sentinel.config;
@ -51,13 +52,53 @@ in {
};
};
age.secrets.influxdb-admin-password = {
generator.script = "alnum";
mode = "440";
group = "influxdb2";
};
age.secrets.influxdb-admin-token = {
generator.script = "alnum";
mode = "440";
group = "influxdb2";
};
age.secrets.influxdb-user-telegraf-token = {
generator.script = "alnum";
mode = "440";
group = "influxdb2";
};
services.influxdb2 = {
enable = true;
settings = {
reporting-disabled = true;
http-bind-address = "${config.meta.wireguard.proxy-sentinel.ipv4}:${toString influxdbPort}";
};
provision = {
enable = true;
initialSetup = {
organization = "default";
bucket = "default";
passwordFile = config.age.secrets.influxdb-admin-password.path;
tokenFile = config.age.secrets.influxdb-admin-token.path;
};
ensureOrganizations = [
{
name = "servers";
}
];
ensureBuckets = [
{
name = "telegraf";
org = "servers";
}
];
};
};
environment.systemPackages = [pkgs.influxdb2-cli];
systemd.services.influxdb2 = {
after = ["sys-subsystem-net-devices-${utils.escapeSystemdPath "proxy-sentinel"}.device"];

9
hosts/zackbiene/default.nix
View file

@ -47,9 +47,12 @@ in {
networking.hosts.${sentinelCfg.meta.wireguard.proxy-sentinel.ipv4} = [sentinelCfg.networking.providedDomains.influxdb];
meta.telegraf = {
enable = true;
influxdb2.domain = sentinelCfg.networking.providedDomains.influxdb;
influxdb2.organization = "servers";
influxdb2.bucket = "telegraf";
influxdb2 = {
domain = sentinelCfg.networking.providedDomains.influxdb;
organization = "servers";
bucket = "telegraf";
node = "ward-influxdb";
};
};
# Fails if there are no SMART devices

1
modules/default.nix
View file

@ -18,6 +18,7 @@
./config/users.nix
./config/xdg.nix
./meta/influxdb-retrieve.nix
./meta/influxdb.nix
./meta/microvms.nix
./meta/nginx.nix

33
modules/meta/influxdb-retrieve.nix Normal file
View file

@ -0,0 +1,33 @@
{
config,
lib,
...
}: let
inherit
(lib)
mkOption
types
;
cfg = config.services.influxdb2;
in {
options.services.influxdb2.provision.retrieveToken = mkOption {
type = types.functionTo (types.functionTo types.str);
readOnly = true;
description = "Script that returns a agenix-rekey generator to retrieve the given token";
default = def: let
id = builtins.substring 0 32 (builtins.hashString "sha256" "${def.user}:${def.org}:${def.name}");
in
{
pkgs,
lib,
...
}: ''
echo " -> Retrieving influxdb token ${def.name} for org ${def.org} on ${config.node.name}" >&2
ssh ${config.node.name} -- \
'bash -c '"'"'influx auth list --json --token "$(< ${cfg.provision.initialSetup.tokenFile})"'"'" \
| ${lib.getExe pkgs.jq} -r '.[] | select(.description | contains("${id}")) | .token' \
|| die "Could not list/find influxdb api token '${def.name}' (${id})"
'';
};
}

64
modules/meta/influxdb.nix
View file

@ -31,9 +31,9 @@
cfg = config.services.influxdb2;
in {
options.services.influxdb2 = {
initialSetup = {
options.services.influxdb2.provision = {
enable = mkEnableOption "initial database setup";
initialSetup = {
organization = mkOption {
type = types.str;
example = "main";
@ -408,7 +408,7 @@ in {
};
};
config = mkIf (cfg.enable && cfg.initialSetup.enable) {
config = mkIf (cfg.enable && cfg.provision.enable) {
assertions = let
validPermissions = flip genAttrs (x: true) [
"authorizations"
@ -435,23 +435,23 @@ in {
"replications"
];
knownOrgs = map (x: x.name) cfg.ensureOrganizations;
knownRemotes = map (x: x.name) cfg.ensureRemotes;
knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.ensureBuckets);
knownOrgs = map (x: x.name) cfg.provision.ensureOrganizations;
knownRemotes = map (x: x.name) cfg.provision.ensureRemotes;
knownBucketsFor = org: map (x: x.name) (filter (x: x.org == org) cfg.provision.ensureBuckets);
in
flip concatMap cfg.ensureBuckets (bucket: [
flip concatMap cfg.provision.ensureBuckets (bucket: [
{
assertion = elem bucket.org knownOrgs;
message = "The influxdb bucket '${bucket.name}' refers to an unknown organization '${bucket.org}'.";
}
])
++ flip concatMap cfg.ensureUsers (user: [
++ flip concatMap cfg.provision.ensureUsers (user: [
{
assertion = elem user.org knownOrgs;
message = "The influxdb user '${user.name}' refers to an unknown organization '${user.org}'.";
}
])
++ flip concatMap cfg.ensureRemotes (remote: [
++ flip concatMap cfg.provision.ensureRemotes (remote: [
{
assertion = (remote.remoteOrgId == null) != (remote.remoteOrg == null);
message = "The influxdb remote '${remote.name}' must specify exactly one of remoteOrgId or remoteOrg.";
@ -461,19 +461,19 @@ in {
message = "The influxdb remote '${remote.name}' refers to an unknown organization '${remote.org}'.";
}
])
++ flip concatMap cfg.ensureReplications (replication: [
++ flip concatMap cfg.provision.ensureReplications (replication: [
{
assertion = elem replication.remote knownRemotes;
message = "The influxdb replication '${replication.name}' refers to an unknown remote '${replication.remote}'.";
}
(let
remote = head (filter (x: x.name == replication.remote) cfg.ensureRemotes);
remote = head (filter (x: x.name == replication.remote) cfg.provision.ensureRemotes);
in {
assertion = elem replication.localBucket (knownBucketsFor remote.org);
message = "The influxdb replication '${replication.name}' refers to an unknown bucket '${replication.localBucket}' in organization '${remote.org}'.";
})
])
++ flip concatMap cfg.ensureApiTokens (apiToken: let
++ flip concatMap cfg.provision.ensureApiTokens (apiToken: let
validBuckets = flip genAttrs (x: true) (knownBucketsFor apiToken.org);
in [
{
@ -557,20 +557,20 @@ in {
# avoid saving the token as the active config.
${influxCli} setup \
--configs-path /dev/null \
--org ${escapeShellArg cfg.initialSetup.organization} \
--bucket ${escapeShellArg cfg.initialSetup.bucket} \
--username ${escapeShellArg cfg.initialSetup.username} \
--password "$(< ${escapeShellArg cfg.initialSetup.passwordFile})" \
--token "$(< ${escapeShellArg cfg.initialSetup.tokenFile})" \
--retention ${escapeShellArg cfg.initialSetup.retention} \
--org ${escapeShellArg cfg.provision.initialSetup.organization} \
--bucket ${escapeShellArg cfg.provision.initialSetup.bucket} \
--username ${escapeShellArg cfg.provision.initialSetup.username} \
--password "$(< ${escapeShellArg cfg.provision.initialSetup.passwordFile})" \
--token "$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})" \
--retention ${escapeShellArg cfg.provision.initialSetup.retention} \
--force >/dev/null
rm -f "$STATE_DIRECTORY/.first_startup"
fi
export INFLUX_TOKEN=$(< ${escapeShellArg cfg.initialSetup.tokenFile})
export INFLUX_TOKEN=$(< ${escapeShellArg cfg.provision.initialSetup.tokenFile})
''
+ flip concatMapStrings cfg.deleteApiTokens (apiToken: ''
+ flip concatMapStrings cfg.provision.deleteApiTokens (apiToken: ''
if id=$(
${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \
| ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id'
@ -579,7 +579,7 @@ in {
echo "Deleted api token id="${escapeShellArg apiToken.id}
fi
'')
+ flip concatMapStrings cfg.deleteReplications (replication: ''
+ flip concatMapStrings cfg.provision.deleteReplications (replication: ''
if id=$(
${influxCli} replication list --json --org ${escapeShellArg replication.org} --name ${escapeShellArg replication.name} 2>/dev/null \
| ${getExe pkgs.jq} -r ".[0].id"
@ -588,7 +588,7 @@ in {
echo "Deleted replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name}
fi
'')
+ flip concatMapStrings cfg.deleteRemotes (remote: ''
+ flip concatMapStrings cfg.provision.deleteRemotes (remote: ''
if id=$(
${influxCli} remote list --json --org ${escapeShellArg remote.org} --name ${escapeShellArg remote.name} 2>/dev/null \
| ${getExe pkgs.jq} -r ".[0].id"
@ -597,7 +597,7 @@ in {
echo "Deleted remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name}
fi
'')
+ flip concatMapStrings cfg.deleteUsers (user: ''
+ flip concatMapStrings cfg.provision.deleteUsers (user: ''
if id=$(
${influxCli} user list --json --name ${escapeShellArg user} 2>/dev/null \
| ${getExe pkgs.jq} -r ".[0].id"
@ -606,7 +606,7 @@ in {
echo "Deleted user name="${escapeShellArg user}
fi
'')
+ flip concatMapStrings cfg.deleteBuckets (bucket: ''
+ flip concatMapStrings cfg.provision.deleteBuckets (bucket: ''
if id=$(
${influxCli} bucket list --json --org ${escapeShellArg bucket.org} --name ${escapeShellArg bucket.name} 2>/dev/null \
| ${getExe pkgs.jq} -r ".[0].id"
@ -615,7 +615,7 @@ in {
echo "Deleted bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name}
fi
'')
+ flip concatMapStrings cfg.deleteOrganizations (org: ''
+ flip concatMapStrings cfg.provision.deleteOrganizations (org: ''
if id=$(
${influxCli} org list --json --name ${escapeShellArg org} 2>/dev/null \
| ${getExe pkgs.jq} -r ".[0].id"
@ -624,7 +624,7 @@ in {
echo "Deleted org name="${escapeShellArg org}
fi
'')
+ flip concatMapStrings cfg.ensureOrganizations (org: let
+ flip concatMapStrings cfg.provision.ensureOrganizations (org: let
listArgs = [
"--name"
org.name
@ -645,7 +645,7 @@ in {
echo "Created org name="${escapeShellArg org.name}
fi
'')
+ flip concatMapStrings cfg.ensureBuckets (bucket: let
+ flip concatMapStrings cfg.provision.ensureBuckets (bucket: let
listArgs = [
"--org"
bucket.org
@ -673,7 +673,7 @@ in {
echo "Created bucket org="${escapeShellArg bucket.org}" name="${escapeShellArg bucket.name}
fi
'')
+ flip concatMapStrings cfg.ensureUsers (user: let
+ flip concatMapStrings cfg.provision.ensureUsers (user: let
listArgs = [
"--name"
user.name
@ -700,7 +700,7 @@ in {
${influxCli} user password ${escapeShellArgs listArgs} \
--password "$(< ${escapeShellArg user.passwordFile})" >/dev/null
'')
+ flip concatMapStrings cfg.ensureRemotes (remote: let
+ flip concatMapStrings cfg.provision.ensureRemotes (remote: let
listArgs = [
"--name"
remote.name
@ -746,7 +746,7 @@ in {
echo "Created remote org="${escapeShellArg remote.org}" name="${escapeShellArg remote.name}
fi
'')
+ flip concatMapStrings cfg.ensureReplications (replication: let
+ flip concatMapStrings cfg.provision.ensureReplications (replication: let
listArgs = [
"--name"
replication.name
@ -780,7 +780,7 @@ in {
echo "Created replication org="${escapeShellArg replication.org}" name="${escapeShellArg replication.name}
fi
'')
+ flip concatMapStrings cfg.ensureApiTokens (apiToken: let
+ flip concatMapStrings cfg.provision.ensureApiTokens (apiToken: let
listArgs = [
"--user"
apiToken.user
@ -801,7 +801,7 @@ in {
if id=$(
${influxCli} auth list --json --org ${escapeShellArg apiToken.org} 2>/dev/null \
| ${getExe pkgs.jq} -r '.[] | select(.description | contains("${apiToken.id}")) | .id'
); then
) && [[ -n "$id" ]]; then
true # No updateable args
else
declare -A bucketIds

67
modules/meta/telegraf.nix
View file

@ -7,7 +7,6 @@
}: let
inherit
(lib)
mdDoc
mkEnableOption
mkIf
mkOption
@ -18,47 +17,71 @@
cfg = config.meta.telegraf;
in {
options.meta.telegraf = {
enable = mkEnableOption (mdDoc "telegraf to push metrics to influx.");
enable = mkEnableOption "telegraf to push metrics to influx.";
scrapeSensors = mkOption {
type = types.bool;
default = true;
description = "Scrape sensors with lm_sensors. You should disable this for virtualized hosts.";
};
influxdb2 = {
domain = mkOption {
type = types.str;
example = "influxdb.example.com";
description = mdDoc "The influxdb v2 database to push to. https will be enforced.";
description = "The influxdb v2 database to push to. https will be enforced.";
};
organization = mkOption {
type = types.str;
description = mdDoc "The organization to push to.";
description = "The organization to push to.";
};
bucket = mkOption {
type = types.str;
description = mdDoc "The bucket to push to.";
description = "The bucket to push to.";
};
user = mkOption {
type = types.str;
default = "admin";
description = "The user for which the api key should be created.";
};
node = mkOption {
type = types.str;
description = "The node which hosts the influxdb service (used to provision an api token).";
};
};
};
config = mkIf cfg.enable {
config = let
tokenDef = {
name = "telegraf (${config.node.name})";
org = "servers";
user = "admin";
readBuckets = ["telegraf"];
writeBuckets = ["telegraf"];
};
in
mkIf cfg.enable {
nodes.${cfg.influxdb2.node}.services.influxdb2.provision.ensureApiTokens = [
tokenDef
];
age.secrets.telegraf-influxdb-token = {
rekeyFile = config.node.secretsDir + "/telegraf-influxdb-token.age";
# TODO generator.script = { pkgs, lib, decrypt, deps, ... }: let
# TODO adminBasicAuth = (builtins.head deps).file;
# TODO adminToken = (builtins.head deps).file; # TODO ..... filter by name?
# TODO in ''
# TODO echo " -> Provisioning influxdb token for telegraf on ${nodeName} at https://${cfg.influxdb2.domain}" >&2
# TODO ${decrypt} ${lib.escapeShellArg aba.file} \
# TODO INFLUX_HOST=https://${aba.host}+${aba.name}:${PW}@${URL}
# TODO | ${pkgs.influxdb2-cli}/bin/influx -niBC 12 ${lib.escapeShellArg host}"+"${lib.escapeShellArg name} \
# TODO || die "Failure"
# TODO '');
generator = {
script = args: ''
echo -n "INFLUX_TOKEN="
${nodes.${cfg.influxdb2.node}.config.services.influxdb2.provision.retrieveToken tokenDef args}
'';
tags = ["influxdb"];
};
mode = "440";
group = "telegraf";
};
# It's intentional to gate this behind smartd. Hosts without smartd are usually
# virtual and won't need sensor collection either.
# XXX: maybe as separate option?
security.elewrap.telegraf-sensors = mkIf config.services.smartd.enable {
security.elewrap.telegraf-sensors = mkIf cfg.scrapeSensors {
command = ["${pkgs.lm_sensors}/bin/sensors" "-A" "-u"];
targetUser = "root";
allowedUsers = ["telegraf"];
@ -159,7 +182,7 @@ in {
systemd.services.telegraf = {
path = [
# Make sensors refer to the correct wrapper
(mkIf config.services.smartd.enable
(mkIf cfg.scrapeSensors
(pkgs.writeShellScriptBin "sensors" config.security.elewrap.telegraf-sensors.path))
];
serviceConfig = {