From c5b28787ef50de3842eb0991aaf131ae6a639abc Mon Sep 17 00:00:00 2001 From: oddlama Date: Sun, 28 Jul 2024 22:16:08 +0200 Subject: [PATCH] feat: configure idmail integration in stalwart --- config/users.nix | 1 - flake.lock | 6 +- hosts/envoy/acme.nix | 2 +- hosts/envoy/idmail.nix | 46 +++- hosts/envoy/stalwart-mail.nix | 488 ++++++++++++++++++++++++++++------ hosts/envoy/stalwart.patch | 76 ++++++ 6 files changed, 525 insertions(+), 94 deletions(-) create mode 100644 hosts/envoy/stalwart.patch diff --git a/config/users.nix b/config/users.nix index eca8b61..3fe82f2 100644 --- a/config/users.nix +++ b/config/users.nix @@ -34,6 +34,5 @@ minecraft = uidGid 975; stalwart-mail = uidGid 974; netbird-home = uidGid 973; - idmail = uidGid 972; }; } diff --git a/flake.lock b/flake.lock index 99cccf1..2b3576b 100644 --- a/flake.lock +++ b/flake.lock @@ -1240,11 +1240,11 @@ "pre-commit-hooks": "pre-commit-hooks_3" }, "locked": { - "lastModified": 1721997987, - "narHash": "sha256-Ck9CSO05AMNFI6e0QH0NnOC02cb8fSZ1r60XVSBBs10=", + "lastModified": 1722122164, + "narHash": "sha256-oMpuAsd/XqDkFjlnawWokObslzeOjt0ivAumrUy5xnM=", "owner": "oddlama", "repo": "idmail", - "rev": "a6516dbdabd7b1473936c9bf79a31ee5515095e6", + "rev": "788f7dbae0f05810892968c74033b5d43c581cc2", "type": "github" }, "original": { diff --git a/hosts/envoy/acme.nix b/hosts/envoy/acme.nix index 65a2986..c91985f 100644 --- a/hosts/envoy/acme.nix +++ b/hosts/envoy/acme.nix @@ -22,7 +22,7 @@ in { }; dnsProvider = "cloudflare"; dnsPropagationCheck = true; - reloadServices = ["nginx" "maddy"]; + reloadServices = ["nginx" "stalwart-mail"]; }; inherit (acme) certs wildcardDomains; }; diff --git a/hosts/envoy/idmail.nix b/hosts/envoy/idmail.nix index 913c0d6..39ce812 100644 --- a/hosts/envoy/idmail.nix +++ b/hosts/envoy/idmail.nix @@ -3,14 +3,15 @@ primaryDomain = mailDomains.primary; idmailDomain = "alias.${primaryDomain}"; in { - environment.persistence."/persist".directories = [ - { - directory = "/var/lib/idmail"; - user = "idmail"; - group = "idmail"; - mode = "0700"; - } - ]; + # Not needed, we store stuff in stalwart's directory + #environment.persistence."/persist".directories = [ + # { + # directory = "/var/lib/idmail"; + # user = "idmail"; + # group = "idmail"; + # mode = "0700"; + # } + #]; globals.services.idmail.domain = idmailDomain; globals.monitoring.http.idmail = { @@ -19,7 +20,34 @@ in { network = "internet"; }; - services.idmail.enable = true; + services.idmail = { + enable = true; + user = "stalwart-mail"; + dataDir = "/var/lib/stalwart-mail"; + provision = { + enable = true; + users.admin = { + admin = true; + # FIXME: 8e8e1c2eb2f1b8c84f1ef294d2fd746b + password_hash = "$argon2id$v=19$m=4096,t=3,p=1$c29tZXJhbmRvbXNhbHQ$Hf0sBCqn5Zp5+7LalZNLKhG0exNsXN2M5T+y3QAjpMM"; + }; + # users.test.password_hash = "$argon2id$v=19$m=4096,t=3,p=1$YXJnbGluYXJsZ2luMjRvaQ$DXdfVNRSFS1QSvJo7OmXIhAYYtT/D92Ku16DiJwxn8U"; + # domains."example.com" = { + # owner = "admin"; + # public = true; + # }; + # mailboxes."me@example.com" = { + # password_hash = "$argon2id$v=19$m=4096,t=3,p=1$YXJnbGluYXJsZ2luMjRvaQ$fiD9Bp3KidVI/E+mGudu6+h9XmF9TU9Bx4VGX0PniDE"; + # owner = "test"; + # api_token = "%{file:${pkgs.writeText "token" token}}%"; + # }; + # aliases."somealias@example.com" = { + # target = "me@example.com"; + # owner = "me@example.com"; + # comment = "Used for xyz"; + # }; + }; + }; systemd.services.idmail.serviceConfig.RestartSec = "60"; # Retry every minute services.nginx = { diff --git a/hosts/envoy/stalwart-mail.nix b/hosts/envoy/stalwart-mail.nix index 3b3b1a4..e907301 100644 --- a/hosts/envoy/stalwart-mail.nix +++ b/hosts/envoy/stalwart-mail.nix @@ -18,6 +18,12 @@ in { } ]; + age.secrets.stalwart-admin-hash = { + rekeyFile = ./secrets/stalwart-admin-hash.age; + mode = "440"; + group = "stalwart-mail"; + }; + users.groups.acme.members = ["stalwart-mail"]; networking.firewall.allowedTCPPorts = [ @@ -38,94 +44,413 @@ in { services.stalwart-mail = { enable = true; + package = pkgs.stalwart-mail.overrideAttrs (old: { + patches = old.patches ++ [./stalwart.patch]; + }); - settings = lib.mkForce { - authentication.fallback-admin = { - user = "admin"; - secret = "$6$tOo2HQnlyAcgyfx5$aMI3uELtqsjkN.gHn8f2W2yxl2ovo.6PU9XxT9jvjJ2CNXpwpumlBq.ZaERPQcTzl4.o1vklB.sdBevXBrLPp0"; + settings = let + case = field: check: value: data: { + "if" = field; + ${check} = value; + "then" = data; }; - - tracer.stdout = { - # Do not use the built-in journal tracer, as it shows much less auxiliary - # information for the same loglevel - type = "stdout"; - level = "info"; - ansi = false; # no colour markers to journald - enable = true; + otherwise = value: {"else" = value;}; + is-smtp = case "listener" "eq" "smtp"; + is-authenticated = data: { + "if" = "!is_empty(authenticated_as)"; + "then" = data; }; - - store.db = { - type = "sqlite"; - path = "${dataDir}/database.sqlite3"; - }; - - storage = { - data = "db"; - fts = "db"; - lookup = "db"; - blob = "db"; - directory = "internal"; - }; - - directory.internal = { - type = "internal"; - store = "db"; - }; - - resolver = { - type = "system"; - public-suffix = [ - "file://${pkgs.publicsuffix-list}/share/publicsuffix/public_suffix_list.dat" - ]; - }; - - config.resource.spam-filter = "file://${config.services.stalwart-mail.package}/etc/stalwart/spamfilter.toml"; - - certificate.default = { - cert = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/fullchain.pem}%"; - private-key = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/key.pem}%"; - default = true; - }; - - server = { - hostname = "mx1.${primaryDomain}"; - tls = { - certificate = "default"; - ignore-client-order = true; + in + lib.mkForce { + authentication.fallback-admin = { + user = "admin"; + secret = "%{file:/run/stalwart-mail/admin-hash}%"; }; - socket = { - nodelay = true; - reuse-addr = true; + + tracer.stdout = { + # Do not use the built-in journal tracer, as it shows much less auxiliary + # information for the same loglevel + type = "stdout"; + level = "info"; + ansi = false; # no colour markers to journald + enable = true; }; - listener = { - smtp = { - protocol = "smtp"; - bind = "[::]:25"; - }; - submissions = { - protocol = "smtp"; - bind = "[::]:465"; - tls.implicit = true; - }; - imaps = { - protocol = "imap"; - bind = "[::]:993"; - tls.implicit = true; - }; - http = { - # jmap, web interface - protocol = "http"; - bind = "[::]:8080"; - url = "https://${stalwartDomain}/jmap"; - }; - sieve = { - protocol = "managesieve"; - bind = "[::]:4190"; - tls.implicit = true; + + store.db = { + type = "sqlite"; + path = "${dataDir}/database.sqlite3"; + }; + + store.idmail = { + type = "sqlite"; + path = "${dataDir}/idmail.db"; + query = let + # Remove comments from SQL and make it single-line + toSingleLineSql = sql: + lib.concatStringsSep " " ( + lib.forEach (lib.flatten (lib.split "\n" sql)) ( + line: lib.optionalString (builtins.match "^[[:space:]]*--.*" line == null) line + ) + ); + in { + # "SELECT name, type, secret, description, quota FROM accounts WHERE name = ?1 AND active = true"; + name = toSingleLineSql '' + SELECT + m.address AS name, + 'individual' AS type, + m.password_hash AS secret, + m.address AS description, + 0 AS quota + FROM mailboxes AS m + JOIN domains AS d ON m.domain = d.domain + JOIN users AS u ON m.owner = u.username + WHERE m.address = ?1 + AND m.active = true + AND d.active = true + AND u.active = true + ''; + # "SELECT member_of FROM group_members WHERE name = ?1"; + members = ""; + # "SELECT name FROM emails WHERE address = ?1"; + recipients = toSingleLineSql '' + -- It is important that we return only one value here, but these three UNIONed + -- queries are guaranteed to be distinct. This is because a mailbox address + -- and alias address can never be the same, their cross-table uniqueness is guaranteed on insert. + -- The catch-all union can also only return something if @domain.tld is given as a parameter, + -- which is invalid for aliases and mailboxes. + + -- Select the primary mailbox address if it matches and + -- all related parts are active + SELECT m.address AS name + FROM mailboxes AS m + JOIN domains AS d ON m.domain = d.domain + JOIN users AS u ON m.owner = u.username + WHERE m.address = ?1 + AND m.active = true + AND d.active = true + AND u.active = true + -- Then select the target of a matching alias + -- but make sure that all related parts are active. + UNION + SELECT a.target AS name + FROM aliases AS a + JOIN domains AS d ON a.domain = d.domain + JOIN ( + -- To check whether the owner is active we need to make a subquery + -- because the owner could be a user or mailbox + SELECT username + FROM users + WHERE active = true + UNION + SELECT m.address AS username + FROM mailboxes AS m + JOIN users AS u ON m.owner = u.username + WHERE m.active = true + AND u.active = true + ) AS u ON a.owner = u.username + WHERE a.address = ?1 + AND a.active = true + AND d.active = true + -- Finally, select any catch_all address that would catch this. + -- Again make sure everything is active. + UNION + SELECT d.catch_all AS name + FROM domains AS d + JOIN mailboxes AS m ON d.catch_all = m.address + JOIN users AS u ON m.owner = u.username + WHERE ?1 = ('@' || d.domain) + AND d.active = true + AND m.active = true + AND u.active = true + + -- This alternative catch-all query would expand catch-alls directly, but would + -- also require sorting the resulting table by precedence and LIMIT 1 + -- to always return just one result. + -- UNION + -- SELECT d.catch_all AS name + -- FROM domains AS d + -- JOIN mailboxes AS m ON d.catch_all = m.address + -- JOIN users AS u ON m.owner = u.username + -- WHERE ?1 LIKE ('%@' || d.domain) + -- AND d.active = true + -- AND m.active = true + -- AND u.active = true + ''; + # "SELECT address FROM emails WHERE name = ?1 AND type != 'list' ORDER BY type DESC, address ASC"; + emails = toSingleLineSql '' + -- Return first the primary address, then any aliases. + SELECT address FROM ( + -- Select primary address, if active + SELECT m.address AS address, 1 AS rowOrder + FROM mailboxes AS m + JOIN domains AS d ON m.domain = d.domain + JOIN users AS u ON m.owner = u.username + WHERE m.address = ?1 + AND m.active = true + AND d.active = true + AND u.active = true + -- Select any active aliases + UNION + SELECT a.address AS address, 2 AS rowOrder + FROM aliases AS a + JOIN domains AS d ON a.domain = d.domain + JOIN ( + -- To check whether the owner is active we need to make a subquery + -- because the owner could be a user or mailbox + SELECT username + FROM users + WHERE active = true + UNION + SELECT m.address AS username + FROM mailboxes AS m + JOIN users AS u ON m.owner = u.username + WHERE m.active = true + AND u.active = true + ) AS u ON a.owner = u.username + WHERE a.target = ?1 + AND a.active = true + AND d.active = true + -- Select the catch-all marker, if we are the target. + UNION + -- Order 2 is correct, it counts as an alias + SELECT ('@' || d.domain) AS address, 2 AS rowOrder + FROM domains AS d + JOIN mailboxes AS m ON d.catch_all = m.address + JOIN users AS u ON m.owner = u.username + WHERE d.catch_all = ?1 + AND d.active = true + AND m.active = true + AND u.active = true + ORDER BY rowOrder, address ASC + ) + ''; + # "SELECT address FROM emails WHERE address LIKE '%' || ?1 || '%' AND type = 'primary' ORDER BY address LIMIT 5"; + verify = toSingleLineSql '' + SELECT m.address AS address + FROM mailboxes AS m + JOIN domains AS d ON m.domain = d.domain + JOIN users AS u ON m.owner = u.username + WHERE m.address LIKE '%' || ?1 || '%' + AND m.active = true + AND d.active = true + AND u.active = true + UNION + SELECT a.address AS address + FROM aliases AS a + JOIN domains AS d ON a.domain = d.domain + JOIN ( + -- To check whether the owner is active we need to make a subquery + -- because the owner could be a user or mailbox + SELECT username + FROM users + WHERE active = true + UNION + SELECT m.address AS username + FROM mailboxes AS m + JOIN users AS u ON m.owner = u.username + WHERE m.active = true + AND u.active = true + ) AS u ON a.owner = u.username + WHERE a.address LIKE '%' || ?1 || '%' + AND a.active = true + AND d.active = true + ORDER BY address + LIMIT 5 + ''; + # "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ?1 AND l.type = 'list' ORDER BY p.address LIMIT 50"; + # XXX: We don't actually expand, but return the same address if it exists since we don't support mailing lists + expand = toSingleLineSql '' + SELECT m.address AS address + FROM mailboxes AS m + JOIN domains AS d ON m.domain = d.domain + JOIN users AS u ON m.owner = u.username + WHERE m.address = ?1 + AND m.active = true + AND d.active = true + AND u.active = true + UNION + SELECT a.address AS address + FROM aliases AS a + JOIN domains AS d ON a.domain = d.domain + JOIN ( + -- To check whether the owner is active we need to make a subquery + -- because the owner could be a user or mailbox + SELECT username + FROM users + WHERE active = true + UNION + SELECT m.address AS username + FROM mailboxes AS m + JOIN users AS u ON m.owner = u.username + WHERE m.active = true + AND u.active = true + ) AS u ON a.owner = u.username + WHERE a.address = ?1 + AND a.active = true + AND d.active = true + ORDER BY address + LIMIT 50 + ''; + # "SELECT 1 FROM emails WHERE address LIKE '%@' || ?1 LIMIT 1"; + domains = toSingleLineSql '' + SELECT domain + FROM domains + WHERE domain = ?1 + ''; }; }; + + storage = { + data = "db"; + fts = "db"; + lookup = "db"; + blob = "db"; + directory = "idmail"; + }; + + directory.idmail = { + type = "sql"; + store = "idmail"; + columns = { + name = "name"; + description = "description"; + secret = "secret"; + email = "email"; + #quota = "quota"; + class = "type"; + }; + }; + + resolver = { + type = "system"; + public-suffix = [ + "file://${pkgs.publicsuffix-list}/share/publicsuffix/public_suffix_list.dat" + ]; + }; + + config.resource.spam-filter = "file://${config.services.stalwart-mail.package}/etc/stalwart/spamfilter.toml"; + + certificate.default = { + cert = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/fullchain.pem}%"; + private-key = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/key.pem}%"; + default = true; + }; + + server = { + hostname = "mx1.${primaryDomain}"; + tls = { + certificate = "default"; + ignore-client-order = true; + }; + socket = { + nodelay = true; + reuse-addr = true; + }; + listener = { + smtp = { + protocol = "smtp"; + bind = "[::]:25"; + }; + submissions = { + protocol = "smtp"; + bind = "[::]:465"; + tls.implicit = true; + }; + imaps = { + protocol = "imap"; + bind = "[::]:993"; + tls.implicit = true; + }; + http = { + # jmap, web interface + protocol = "http"; + bind = "[::]:8080"; + url = "https://${stalwartDomain}/jmap"; + }; + sieve = { + protocol = "managesieve"; + bind = "[::]:4190"; + tls.implicit = true; + }; + }; + }; + + #queue.outbound.next-hop = [ + # (case "rcpt_domain" "in-list" "default/domains" "local") + # (otherwise false) + #]; + + #queue.schedule = { + # retry = ["2m" "5m" "10m" "15m" "30m" "1h" "2h"]; + # notify = ["1d" "3d"]; + # expire = "5d"; + #}; + + # XXX: needed? jmap.directory = "idmail"; + + imap = { + request.max-size = 52428800; + auth = { + max-failures = 3; + allow-plain-text = false; + }; + timeout = { + authentication = "30m"; + anonymous = "1m"; + idle = "30m"; + }; + rate-limit = { + requests = "2000/1m"; + concurrent = 4; + }; + }; + + session.extensions = { + pipelining = true; + chunking = true; + requiretls = true; + no-soliciting = ""; + dsn = false; + expn = [ + (is-authenticated true) + (otherwise false) + ]; + vrfy = [ + (is-authenticated true) + (otherwise false) + ]; + future-release = [ + (is-authenticated "30d") + (otherwise false) + ]; + deliver-by = [ + (is-authenticated "365d") + (otherwise false) + ]; + mt-priority = [ + (is-authenticated "mixer") + (otherwise false) + ]; + }; + + session.ehlo = { + require = true; + reject-non-fqdn = [ + (is-smtp true) + (otherwise false) + ]; + }; + + session.rcpt = { + # XXX: needed? directory = "idmail"; + catch-all = true; + relay = [ + (is-authenticated true) + (otherwise false) + ]; + max-recipients = 25; + }; }; - }; }; services.nginx = { @@ -156,6 +481,9 @@ in { in { preStart = lib.mkAfter '' cat ${configFile} > /run/stalwart-mail/config.toml + ${pkgs.gnugrep}/bin/grep -v '^\s*$\|^\s*#' \ + < ${config.age.secrets.stalwart-admin-hash.path} \ + | tr -d '\n' > /run/stalwart-mail/admin-hash ''; serviceConfig = { RuntimeDirectory = "stalwart-mail"; diff --git a/hosts/envoy/stalwart.patch b/hosts/envoy/stalwart.patch new file mode 100644 index 0000000..9b9e420 --- /dev/null +++ b/hosts/envoy/stalwart.patch @@ -0,0 +1,76 @@ +diff --git a/crates/directory/src/backend/sql/lookup.rs b/crates/directory/src/backend/sql/lookup.rs +index e49c5dab..18c9005e 100644 +--- a/crates/directory/src/backend/sql/lookup.rs ++++ b/crates/directory/src/backend/sql/lookup.rs +@@ -25,6 +25,7 @@ impl SqlDirectory { + QueryBy::Name(username) => { + account_name = username.to_string(); + ++ tracing::info!(context = "directory", event = "sql lookup", username = username, "Name -> query_name"); + self.store + .query::(&self.mappings.query_name, vec![username.into()]) + .await? +@@ -37,6 +38,7 @@ impl SqlDirectory { + } + account_id = Some(uid); + ++ tracing::info!(context = "directory", event = "sql lookup", username = account_name, "Id -> query_name"); + self.store + .query::( + &self.mappings.query_name, +@@ -53,6 +55,7 @@ impl SqlDirectory { + account_name = username.to_string(); + secret = secret_.into(); + ++ tracing::info!(context = "directory", event = "sql lookup", username = username, "Credentials -> query_name"); + self.store + .query::(&self.mappings.query_name, vec![username.into()]) + .await? +@@ -112,6 +115,7 @@ impl SqlDirectory { + + // Obtain emails + if !self.mappings.query_emails.is_empty() { ++ tracing::info!(context = "directory", event = "sql lookup", principal_name = principal.name, "Query emails"); + principal.emails = self + .store + .query::( +@@ -126,6 +130,7 @@ impl SqlDirectory { + } + + pub async fn email_to_ids(&self, address: &str) -> crate::Result> { ++ tracing::info!(context = "directory", event = "sql lookup", address = address, "Query recipients"); + let names = self + .store + .query::(&self.mappings.query_recipients, vec![address.into()]) +@@ -143,6 +148,7 @@ impl SqlDirectory { + } + + pub async fn rcpt(&self, address: &str) -> crate::Result { ++ tracing::info!(context = "directory", event = "sql lookup", address = address, "Query rcpt"); + self.store + .query::( + &self.mappings.query_recipients, +@@ -153,6 +159,7 @@ impl SqlDirectory { + } + + pub async fn vrfy(&self, address: &str) -> crate::Result> { ++ tracing::info!(context = "directory", event = "sql lookup", address = address, "Query vrfy"); + self.store + .query::( + &self.mappings.query_verify, +@@ -164,6 +171,7 @@ impl SqlDirectory { + } + + pub async fn expn(&self, address: &str) -> crate::Result> { ++ tracing::info!(context = "directory", event = "sql lookup", address = address, "Query expn"); + self.store + .query::( + &self.mappings.query_expand, +@@ -175,6 +183,7 @@ impl SqlDirectory { + } + + pub async fn is_local_domain(&self, domain: &str) -> crate::Result { ++ tracing::info!(context = "directory", event = "sql lookup", domain = domain, "Query is local domain"); + self.store + .query::(&self.mappings.query_domains, vec![domain.into()]) + .await