feat: configure idmail integration in stalwart

config/users.nix

@@ -34,6 +34,5 @@
     minecraft = uidGid 975;
     stalwart-mail = uidGid 974;
     netbird-home = uidGid 973;
-    idmail = uidGid 972;
   };
 }

flake.lock

@@ -1240,11 +1240,11 @@
         "pre-commit-hooks": "pre-commit-hooks_3"
       },
       "locked": {
-        "lastModified": 1721997987,
+        "lastModified": 1722122164,
-        "narHash": "sha256-Ck9CSO05AMNFI6e0QH0NnOC02cb8fSZ1r60XVSBBs10=",
+        "narHash": "sha256-oMpuAsd/XqDkFjlnawWokObslzeOjt0ivAumrUy5xnM=",
         "owner": "oddlama",
         "repo": "idmail",
-        "rev": "a6516dbdabd7b1473936c9bf79a31ee5515095e6",
+        "rev": "788f7dbae0f05810892968c74033b5d43c581cc2",
         "type": "github"
       },
       "original": {

hosts/envoy/acme.nix

@@ -22,7 +22,7 @@ in {
       };
       dnsProvider = "cloudflare";
       dnsPropagationCheck = true;
-      reloadServices = ["nginx" "maddy"];
+      reloadServices = ["nginx" "stalwart-mail"];
     };
     inherit (acme) certs wildcardDomains;
   };

hosts/envoy/idmail.nix

@@ -3,14 +3,15 @@
   primaryDomain = mailDomains.primary;
   idmailDomain = "alias.${primaryDomain}";
 in {
-  environment.persistence."/persist".directories = [
+  # Not needed, we store stuff in stalwart's directory
-    {
+  #environment.persistence."/persist".directories = [
-      directory = "/var/lib/idmail";
+  #  {
-      user = "idmail";
+  #    directory = "/var/lib/idmail";
-      group = "idmail";
+  #    user = "idmail";
-      mode = "0700";
+  #    group = "idmail";
-    }
+  #    mode = "0700";
-  ];
+  #  }
+  #];
 
   globals.services.idmail.domain = idmailDomain;
   globals.monitoring.http.idmail = {
@@ -19,7 +20,34 @@ in {
     network = "internet";
   };
 
-  services.idmail.enable = true;
+  services.idmail = {
+    enable = true;
+    user = "stalwart-mail";
+    dataDir = "/var/lib/stalwart-mail";
+    provision = {
+      enable = true;
+      users.admin = {
+        admin = true;
+        # FIXME: 8e8e1c2eb2f1b8c84f1ef294d2fd746b
+        password_hash = "$argon2id$v=19$m=4096,t=3,p=1$c29tZXJhbmRvbXNhbHQ$Hf0sBCqn5Zp5+7LalZNLKhG0exNsXN2M5T+y3QAjpMM";
+      };
+      # users.test.password_hash = "$argon2id$v=19$m=4096,t=3,p=1$YXJnbGluYXJsZ2luMjRvaQ$DXdfVNRSFS1QSvJo7OmXIhAYYtT/D92Ku16DiJwxn8U";
+      # domains."example.com" = {
+      #   owner = "admin";
+      #   public = true;
+      # };
+      # mailboxes."me@example.com" = {
+      #   password_hash = "$argon2id$v=19$m=4096,t=3,p=1$YXJnbGluYXJsZ2luMjRvaQ$fiD9Bp3KidVI/E+mGudu6+h9XmF9TU9Bx4VGX0PniDE";
+      #   owner = "test";
+      #   api_token = "%{file:${pkgs.writeText "token" token}}%";
+      # };
+      # aliases."somealias@example.com" = {
+      #   target = "me@example.com";
+      #   owner = "me@example.com";
+      #   comment = "Used for xyz";
+      # };
+    };
+  };
   systemd.services.idmail.serviceConfig.RestartSec = "60"; # Retry every minute
 
   services.nginx = {

hosts/envoy/stalwart-mail.nix

@@ -18,6 +18,12 @@ in {
     }
   ];
 
+  age.secrets.stalwart-admin-hash = {
+    rekeyFile = ./secrets/stalwart-admin-hash.age;
+    mode = "440";
+    group = "stalwart-mail";
+  };
+
   users.groups.acme.members = ["stalwart-mail"];
 
   networking.firewall.allowedTCPPorts = [
@@ -38,94 +44,413 @@ in {
 
   services.stalwart-mail = {
     enable = true;
+    package = pkgs.stalwart-mail.overrideAttrs (old: {
+      patches = old.patches ++ [./stalwart.patch];
+    });
 
-    settings = lib.mkForce {
+    settings = let
-      authentication.fallback-admin = {
+      case = field: check: value: data: {
-        user = "admin";
+        "if" = field;
-        secret = "$6$tOo2HQnlyAcgyfx5$aMI3uELtqsjkN.gHn8f2W2yxl2ovo.6PU9XxT9jvjJ2CNXpwpumlBq.ZaERPQcTzl4.o1vklB.sdBevXBrLPp0";
+        ${check} = value;
+        "then" = data;
       };
-
+      otherwise = value: {"else" = value;};
-      tracer.stdout = {
+      is-smtp = case "listener" "eq" "smtp";
-        # Do not use the built-in journal tracer, as it shows much less auxiliary
+      is-authenticated = data: {
-        # information for the same loglevel
+        "if" = "!is_empty(authenticated_as)";
-        type = "stdout";
+        "then" = data;
-        level = "info";
-        ansi = false; # no colour markers to journald
-        enable = true;
       };
-
+    in
-      store.db = {
+      lib.mkForce {
-        type = "sqlite";
+        authentication.fallback-admin = {
-        path = "${dataDir}/database.sqlite3";
+          user = "admin";
-      };
+          secret = "%{file:/run/stalwart-mail/admin-hash}%";
-
-      storage = {
-        data = "db";
-        fts = "db";
-        lookup = "db";
-        blob = "db";
-        directory = "internal";
-      };
-
-      directory.internal = {
-        type = "internal";
-        store = "db";
-      };
-
-      resolver = {
-        type = "system";
-        public-suffix = [
-          "file://${pkgs.publicsuffix-list}/share/publicsuffix/public_suffix_list.dat"
-        ];
-      };
-
-      config.resource.spam-filter = "file://${config.services.stalwart-mail.package}/etc/stalwart/spamfilter.toml";
-
-      certificate.default = {
-        cert = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/fullchain.pem}%";
-        private-key = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/key.pem}%";
-        default = true;
-      };
-
-      server = {
-        hostname = "mx1.${primaryDomain}";
-        tls = {
-          certificate = "default";
-          ignore-client-order = true;
         };
-        socket = {
+
-          nodelay = true;
+        tracer.stdout = {
-          reuse-addr = true;
+          # Do not use the built-in journal tracer, as it shows much less auxiliary
+          # information for the same loglevel
+          type = "stdout";
+          level = "info";
+          ansi = false; # no colour markers to journald
+          enable = true;
         };
-        listener = {
+
-          smtp = {
+        store.db = {
-            protocol = "smtp";
+          type = "sqlite";
-            bind = "[::]:25";
+          path = "${dataDir}/database.sqlite3";
-          };
+        };
-          submissions = {
+
-            protocol = "smtp";
+        store.idmail = {
-            bind = "[::]:465";
+          type = "sqlite";
-            tls.implicit = true;
+          path = "${dataDir}/idmail.db";
-          };
+          query = let
-          imaps = {
+            # Remove comments from SQL and make it single-line
-            protocol = "imap";
+            toSingleLineSql = sql:
-            bind = "[::]:993";
+              lib.concatStringsSep " " (
-            tls.implicit = true;
+                lib.forEach (lib.flatten (lib.split "\n" sql)) (
-          };
+                  line: lib.optionalString (builtins.match "^[[:space:]]*--.*" line == null) line
-          http = {
+                )
-            # jmap, web interface
+              );
-            protocol = "http";
+          in {
-            bind = "[::]:8080";
+            # "SELECT name, type, secret, description, quota FROM accounts WHERE name = ?1 AND active = true";
-            url = "https://${stalwartDomain}/jmap";
+            name = toSingleLineSql ''
-          };
+              SELECT
-          sieve = {
+                  m.address AS name,
-            protocol = "managesieve";
+                  'individual' AS type,
-            bind = "[::]:4190";
+                  m.password_hash AS secret,
-            tls.implicit = true;
+                  m.address AS description,
+                  0 AS quota
+                FROM mailboxes AS m
+                JOIN domains AS d ON m.domain = d.domain
+                JOIN users AS u ON m.owner = u.username
+                WHERE m.address = ?1
+                  AND m.active = true
+                  AND d.active = true
+                  AND u.active = true
+            '';
+            # "SELECT member_of FROM group_members WHERE name = ?1";
+            members = "";
+            # "SELECT name FROM emails WHERE address = ?1";
+            recipients = toSingleLineSql ''
+              -- It is important that we return only one value here, but these three UNIONed
+              -- queries are guaranteed to be distinct. This is because a mailbox address
+              -- and alias address can never be the same, their cross-table uniqueness is guaranteed on insert.
+              -- The catch-all union can also only return something if @domain.tld is given as a parameter,
+              -- which is invalid for aliases and mailboxes.
+
+              -- Select the primary mailbox address if it matches and
+              -- all related parts are active
+              SELECT m.address AS name
+                FROM mailboxes AS m
+                JOIN domains AS d ON m.domain = d.domain
+                JOIN users AS u ON m.owner = u.username
+                WHERE m.address = ?1
+                  AND m.active = true
+                  AND d.active = true
+                  AND u.active = true
+              -- Then select the target of a matching alias
+              -- but make sure that all related parts are active.
+              UNION
+              SELECT a.target AS name
+                FROM aliases AS a
+                JOIN domains AS d ON a.domain = d.domain
+                JOIN (
+                  -- To check whether the owner is active we need to make a subquery
+                  -- because the owner could be a user or mailbox
+                  SELECT username
+                    FROM users
+                    WHERE active = true
+                  UNION
+                  SELECT m.address AS username
+                    FROM mailboxes AS m
+                    JOIN users AS u ON m.owner = u.username
+                    WHERE m.active = true
+                      AND u.active = true
+                ) AS u ON a.owner = u.username
+                WHERE a.address = ?1
+                  AND a.active = true
+                  AND d.active = true
+              -- Finally, select any catch_all address that would catch this.
+              -- Again make sure everything is active.
+              UNION
+              SELECT d.catch_all AS name
+                FROM domains AS d
+                JOIN mailboxes AS m ON d.catch_all = m.address
+                JOIN users AS u ON m.owner = u.username
+                WHERE ?1 = ('@' || d.domain)
+                  AND d.active = true
+                  AND m.active = true
+                  AND u.active = true
+
+              -- This alternative catch-all query would expand catch-alls directly, but would
+              -- also require sorting the resulting table by precedence and LIMIT 1
+              -- to always return just one result.
+              -- UNION
+              -- SELECT d.catch_all AS name
+              --   FROM domains AS d
+              --   JOIN mailboxes AS m ON d.catch_all = m.address
+              --   JOIN users AS u ON m.owner = u.username
+              --   WHERE ?1 LIKE ('%@' || d.domain)
+              --     AND d.active = true
+              --     AND m.active = true
+              --     AND u.active = true
+            '';
+            # "SELECT address FROM emails WHERE name = ?1 AND type != 'list' ORDER BY type DESC, address ASC";
+            emails = toSingleLineSql ''
+              -- Return first the primary address, then any aliases.
+              SELECT address FROM (
+                -- Select primary address, if active
+                SELECT m.address AS address, 1 AS rowOrder
+                  FROM mailboxes AS m
+                  JOIN domains AS d ON m.domain = d.domain
+                  JOIN users AS u ON m.owner = u.username
+                  WHERE m.address = ?1
+                    AND m.active = true
+                    AND d.active = true
+                    AND u.active = true
+                -- Select any active aliases
+                UNION
+                SELECT a.address AS address, 2 AS rowOrder
+                  FROM aliases AS a
+                  JOIN domains AS d ON a.domain = d.domain
+                  JOIN (
+                    -- To check whether the owner is active we need to make a subquery
+                    -- because the owner could be a user or mailbox
+                    SELECT username
+                      FROM users
+                      WHERE active = true
+                    UNION
+                    SELECT m.address AS username
+                      FROM mailboxes AS m
+                      JOIN users AS u ON m.owner = u.username
+                      WHERE m.active = true
+                        AND u.active = true
+                  ) AS u ON a.owner = u.username
+                  WHERE a.target = ?1
+                    AND a.active = true
+                    AND d.active = true
+                -- Select the catch-all marker, if we are the target.
+                UNION
+                -- Order 2 is correct, it counts as an alias
+                SELECT ('@' || d.domain) AS address, 2 AS rowOrder
+                  FROM domains AS d
+                  JOIN mailboxes AS m ON d.catch_all = m.address
+                  JOIN users AS u ON m.owner = u.username
+                  WHERE d.catch_all = ?1
+                    AND d.active = true
+                    AND m.active = true
+                    AND u.active = true
+                ORDER BY rowOrder, address ASC
+              )
+            '';
+            # "SELECT address FROM emails WHERE address LIKE '%' || ?1 || '%' AND type = 'primary' ORDER BY address LIMIT 5";
+            verify = toSingleLineSql ''
+              SELECT m.address AS address
+                FROM mailboxes AS m
+                JOIN domains AS d ON m.domain = d.domain
+                JOIN users AS u ON m.owner = u.username
+                WHERE m.address LIKE '%' || ?1 || '%'
+                  AND m.active = true
+                  AND d.active = true
+                  AND u.active = true
+              UNION
+              SELECT a.address AS address
+                FROM aliases AS a
+                JOIN domains AS d ON a.domain = d.domain
+                JOIN (
+                  -- To check whether the owner is active we need to make a subquery
+                  -- because the owner could be a user or mailbox
+                  SELECT username
+                    FROM users
+                    WHERE active = true
+                  UNION
+                  SELECT m.address AS username
+                    FROM mailboxes AS m
+                    JOIN users AS u ON m.owner = u.username
+                    WHERE m.active = true
+                      AND u.active = true
+                ) AS u ON a.owner = u.username
+                WHERE a.address LIKE '%' || ?1 || '%'
+                  AND a.active = true
+                  AND d.active = true
+              ORDER BY address
+              LIMIT 5
+            '';
+            # "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ?1 AND l.type = 'list' ORDER BY p.address LIMIT 50";
+            # XXX: We don't actually expand, but return the same address if it exists since we don't support mailing lists
+            expand = toSingleLineSql ''
+              SELECT m.address AS address
+                FROM mailboxes AS m
+                JOIN domains AS d ON m.domain = d.domain
+                JOIN users AS u ON m.owner = u.username
+                WHERE m.address = ?1
+                  AND m.active = true
+                  AND d.active = true
+                  AND u.active = true
+              UNION
+              SELECT a.address AS address
+                FROM aliases AS a
+                JOIN domains AS d ON a.domain = d.domain
+                JOIN (
+                  -- To check whether the owner is active we need to make a subquery
+                  -- because the owner could be a user or mailbox
+                  SELECT username
+                    FROM users
+                    WHERE active = true
+                  UNION
+                  SELECT m.address AS username
+                    FROM mailboxes AS m
+                    JOIN users AS u ON m.owner = u.username
+                    WHERE m.active = true
+                      AND u.active = true
+                ) AS u ON a.owner = u.username
+                WHERE a.address = ?1
+                  AND a.active = true
+                  AND d.active = true
+              ORDER BY address
+              LIMIT 50
+            '';
+            # "SELECT 1 FROM emails WHERE address LIKE '%@' || ?1 LIMIT 1";
+            domains = toSingleLineSql ''
+              SELECT domain
+                FROM domains
+                WHERE domain = ?1
+            '';
           };
         };
+
+        storage = {
+          data = "db";
+          fts = "db";
+          lookup = "db";
+          blob = "db";
+          directory = "idmail";
+        };
+
+        directory.idmail = {
+          type = "sql";
+          store = "idmail";
+          columns = {
+            name = "name";
+            description = "description";
+            secret = "secret";
+            email = "email";
+            #quota = "quota";
+            class = "type";
+          };
+        };
+
+        resolver = {
+          type = "system";
+          public-suffix = [
+            "file://${pkgs.publicsuffix-list}/share/publicsuffix/public_suffix_list.dat"
+          ];
+        };
+
+        config.resource.spam-filter = "file://${config.services.stalwart-mail.package}/etc/stalwart/spamfilter.toml";
+
+        certificate.default = {
+          cert = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/fullchain.pem}%";
+          private-key = "%{file:${config.security.acme.certs.${primaryDomain}.directory}/key.pem}%";
+          default = true;
+        };
+
+        server = {
+          hostname = "mx1.${primaryDomain}";
+          tls = {
+            certificate = "default";
+            ignore-client-order = true;
+          };
+          socket = {
+            nodelay = true;
+            reuse-addr = true;
+          };
+          listener = {
+            smtp = {
+              protocol = "smtp";
+              bind = "[::]:25";
+            };
+            submissions = {
+              protocol = "smtp";
+              bind = "[::]:465";
+              tls.implicit = true;
+            };
+            imaps = {
+              protocol = "imap";
+              bind = "[::]:993";
+              tls.implicit = true;
+            };
+            http = {
+              # jmap, web interface
+              protocol = "http";
+              bind = "[::]:8080";
+              url = "https://${stalwartDomain}/jmap";
+            };
+            sieve = {
+              protocol = "managesieve";
+              bind = "[::]:4190";
+              tls.implicit = true;
+            };
+          };
+        };
+
+        #queue.outbound.next-hop = [
+        #  (case "rcpt_domain" "in-list" "default/domains" "local")
+        #  (otherwise false)
+        #];
+
+        #queue.schedule = {
+        #  retry = ["2m" "5m" "10m" "15m" "30m" "1h" "2h"];
+        #  notify = ["1d" "3d"];
+        #  expire = "5d";
+        #};
+
+        # XXX: needed? jmap.directory = "idmail";
+
+        imap = {
+          request.max-size = 52428800;
+          auth = {
+            max-failures = 3;
+            allow-plain-text = false;
+          };
+          timeout = {
+            authentication = "30m";
+            anonymous = "1m";
+            idle = "30m";
+          };
+          rate-limit = {
+            requests = "2000/1m";
+            concurrent = 4;
+          };
+        };
+
+        session.extensions = {
+          pipelining = true;
+          chunking = true;
+          requiretls = true;
+          no-soliciting = "";
+          dsn = false;
+          expn = [
+            (is-authenticated true)
+            (otherwise false)
+          ];
+          vrfy = [
+            (is-authenticated true)
+            (otherwise false)
+          ];
+          future-release = [
+            (is-authenticated "30d")
+            (otherwise false)
+          ];
+          deliver-by = [
+            (is-authenticated "365d")
+            (otherwise false)
+          ];
+          mt-priority = [
+            (is-authenticated "mixer")
+            (otherwise false)
+          ];
+        };
+
+        session.ehlo = {
+          require = true;
+          reject-non-fqdn = [
+            (is-smtp true)
+            (otherwise false)
+          ];
+        };
+
+        session.rcpt = {
+          # XXX: needed? directory = "idmail";
+          catch-all = true;
+          relay = [
+            (is-authenticated true)
+            (otherwise false)
+          ];
+          max-recipients = 25;
+        };
       };
-    };
   };
 
   services.nginx = {
@@ -156,6 +481,9 @@ in {
   in {
     preStart = lib.mkAfter ''
       cat ${configFile} > /run/stalwart-mail/config.toml
+      ${pkgs.gnugrep}/bin/grep -v '^\s*$\|^\s*#' \
+        < ${config.age.secrets.stalwart-admin-hash.path} \
+        | tr -d '\n' > /run/stalwart-mail/admin-hash
     '';
     serviceConfig = {
       RuntimeDirectory = "stalwart-mail";

hosts/envoy/stalwart.patch

@@ -0,0 +1,76 @@
+diff --git a/crates/directory/src/backend/sql/lookup.rs b/crates/directory/src/backend/sql/lookup.rs
+index e49c5dab..18c9005e 100644
+--- a/crates/directory/src/backend/sql/lookup.rs
++++ b/crates/directory/src/backend/sql/lookup.rs
+@@ -25,6 +25,7 @@ impl SqlDirectory {
+             QueryBy::Name(username) => {
+                 account_name = username.to_string();
+ 
++                tracing::info!(context = "directory", event = "sql lookup", username = username, "Name -> query_name");
+                 self.store
+                     .query::<NamedRows>(&self.mappings.query_name, vec![username.into()])
+                     .await?
+@@ -37,6 +38,7 @@ impl SqlDirectory {
+                 }
+                 account_id = Some(uid);
+ 
++                tracing::info!(context = "directory", event = "sql lookup", username = account_name, "Id -> query_name");
+                 self.store
+                     .query::<NamedRows>(
+                         &self.mappings.query_name,
+@@ -53,6 +55,7 @@ impl SqlDirectory {
+                 account_name = username.to_string();
+                 secret = secret_.into();
+ 
++                tracing::info!(context = "directory", event = "sql lookup", username = username, "Credentials -> query_name");
+                 self.store
+                     .query::<NamedRows>(&self.mappings.query_name, vec![username.into()])
+                     .await?
+@@ -112,6 +115,7 @@ impl SqlDirectory {
+ 
+         // Obtain emails
+         if !self.mappings.query_emails.is_empty() {
++            tracing::info!(context = "directory", event = "sql lookup", principal_name = principal.name, "Query emails");
+             principal.emails = self
+                 .store
+                 .query::<Rows>(
+@@ -126,6 +130,7 @@ impl SqlDirectory {
+     }
+ 
+     pub async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> {
++        tracing::info!(context = "directory", event = "sql lookup", address = address, "Query recipients");
+         let names = self
+             .store
+             .query::<Rows>(&self.mappings.query_recipients, vec![address.into()])
+@@ -143,6 +148,7 @@ impl SqlDirectory {
+     }
+ 
+     pub async fn rcpt(&self, address: &str) -> crate::Result<bool> {
++        tracing::info!(context = "directory", event = "sql lookup", address = address, "Query rcpt");
+         self.store
+             .query::<bool>(
+                 &self.mappings.query_recipients,
+@@ -153,6 +159,7 @@ impl SqlDirectory {
+     }
+ 
+     pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> {
++        tracing::info!(context = "directory", event = "sql lookup", address = address, "Query vrfy");
+         self.store
+             .query::<Rows>(
+                 &self.mappings.query_verify,
+@@ -164,6 +171,7 @@ impl SqlDirectory {
+     }
+ 
+     pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> {
++        tracing::info!(context = "directory", event = "sql lookup", address = address, "Query expn");
+         self.store
+             .query::<Rows>(
+                 &self.mappings.query_expand,
+@@ -175,6 +183,7 @@ impl SqlDirectory {
+     }
+ 
+     pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> {
++        tracing::info!(context = "directory", event = "sql lookup", domain = domain, "Query is local domain");
+         self.store
+             .query::<bool>(&self.mappings.query_domains, vec![domain.into()])
+             .await