feat: fix DNS entry listing in stalwart via patch, increase concurrent imap limit, allow aliasing existing mailboxes

hosts/envoy/a.patch

@@ -0,0 +1,27 @@
+diff --git a/crates/jmap/src/api/management/domain.rs b/crates/jmap/src/api/management/domain.rs
+index e3890df5..7083aaf6 100644
+--- a/crates/jmap/src/api/management/domain.rs
++++ b/crates/jmap/src/api/management/domain.rs
+@@ -123,6 +123,8 @@ impl JMAP {
+     }
+ 
+     async fn build_dns_records(&self, domain_name: &str) -> trc::Result<Vec<DnsRecord>> {
++        let signature_config = self.core.storage.config.build_config("signature").await?;
++
+         // Obtain server name
+         let server_name = self
+             .core
+@@ -143,7 +145,11 @@ impl JMAP {
+                 }
+                 _ => (),
+             }
+-            keys.insert(key, value);
++            let val = signature_config.keys
++                .get(&format!("signature.{key}"))
++                .cloned()
++                .unwrap_or(value.clone());
++            keys.insert(key, val);
+         }
+ 
+         // Add MX and CNAME records
+

hosts/envoy/stalwart-mail.nix

@@ -50,6 +50,14 @@ in {
 
   services.stalwart-mail = {
     enable = true;
+    package = pkgs.stalwart-mail.overrideAttrs (old: {
+      patches =
+        old.patches
+        ++ [
+          ./a.patch
+        ];
+      doCheck = false;
+    });
     settings = let
       case = field: check: value: data: {
         "if" = field;
@@ -139,26 +147,20 @@ in {
             members = "";
             # "SELECT name FROM emails WHERE address = ?1";
             recipients = toSingleLineSql ''
-              -- It is important that we return only one value here, but these three UNIONed
+              -- It is important that we return only one value here, but in theory three UNIONed
               -- queries are guaranteed to be distinct. This is because a mailbox address
               -- and alias address can never be the same, their cross-table uniqueness is guaranteed on insert.
               -- The catch-all union can also only return something if @domain.tld is given as a parameter,
               -- which is invalid for aliases and mailboxes.
-
+              --
-              -- Select the primary mailbox address if it matches and
+              -- Nonetheless, it may be beneficial to allow an alias to override an existing mailbox,
-              -- all related parts are active
+              -- so we can have send-only mailboxes which have their incoming mail redirected somewhere else.
-              SELECT m.address AS name
+              -- Therefore, we make sure to order the query by (aliases -> mailboxes -> catch all) and only return the
-                FROM mailboxes AS m
+              -- highest priority one.
-                JOIN domains AS d ON m.domain = d.domain
+              SELECT name FROM (
-                JOIN users AS u ON m.owner = u.username
+                -- Select the target of a matching alias (if any)
-                WHERE m.address = ?1
-                  AND m.active = true
-                  AND d.active = true
-                  AND u.active = true
-              -- Then select the target of a matching alias
                 -- but make sure that all related parts are active.
-              UNION
+                SELECT a.target AS name, 1 AS rowOrder
-              SELECT a.target AS name
                   FROM aliases AS a
                   JOIN domains AS d ON a.domain = d.domain
                   JOIN (
@@ -177,10 +179,21 @@ in {
                   WHERE a.address = ?1
                     AND a.active = true
                     AND d.active = true
+                -- Select the primary mailbox address if it matches and
+                -- all related parts are active.
+                UNION
+                SELECT m.address AS name, 2 AS rowOrder
+                  FROM mailboxes AS m
+                  JOIN domains AS d ON m.domain = d.domain
+                  JOIN users AS u ON m.owner = u.username
+                  WHERE m.address = ?1
+                    AND m.active = true
+                    AND d.active = true
+                    AND u.active = true
                 -- Finally, select any catch_all address that would catch this.
                 -- Again make sure everything is active.
                 UNION
-              SELECT d.catch_all AS name
+                SELECT d.catch_all, 3 AS rowOrder AS name
                   FROM domains AS d
                   JOIN mailboxes AS m ON d.catch_all = m.address
                   JOIN users AS u ON m.owner = u.username
@@ -188,19 +201,9 @@ in {
                     AND d.active = true
                     AND m.active = true
                     AND u.active = true
-
+                ORDER BY rowOrder, name ASC
-              -- This alternative catch-all query would expand catch-alls directly, but would
+                LIMIT 1
-              -- also require sorting the resulting table by precedence and LIMIT 1
+              )
-              -- to always return just one result.
-              -- UNION
-              -- SELECT d.catch_all AS name
-              --   FROM domains AS d
-              --   JOIN mailboxes AS m ON d.catch_all = m.address
-              --   JOIN users AS u ON m.owner = u.username
-              --   WHERE ?1 LIKE ('%@' || d.domain)
-              --     AND d.active = true
-              --     AND m.active = true
-              --     AND u.active = true
             '';
             # "SELECT address FROM emails WHERE name = ?1 AND type != 'list' ORDER BY type DESC, address ASC";
             emails = toSingleLineSql ''
@@ -414,8 +417,8 @@ in {
             idle = "30m";
           };
           rate-limit = {
-            requests = "2000/1m";
+            requests = "20000/1m";
-            concurrent = 4;
+            concurrent = 32;
           };
         };
 
@@ -543,7 +546,7 @@ in {
         lib.forEach (builtins.attrNames globals.mail.domains) (domain: ''
           if [[ ! -e /var/lib/stalwart-mail/dkim/rsa-${domain}.key ]]; then
             echo "Generating DKIM key for ${domain} (rsa)"
-            ${lib.getExe pkgs.openssl} genrsa -out /var/lib/stalwart-mail/dkim/rsa-${domain}.key 2048
+            ${lib.getExe pkgs.openssl} genrsa -traditional -out /var/lib/stalwart-mail/dkim/rsa-${domain}.key 2048
           fi
           if [[ ! -e /var/lib/stalwart-mail/dkim/ed25519-${domain}.key ]]; then
             echo "Generating DKIM key for ${domain} (ed25519)"
@@ -563,4 +566,37 @@ in {
       RestartSec = "60"; # Retry every minute
     };
   };
+
+  # systemd.services.stalwart-backup = {
+  #   description = "Stalwart and idmail backup";
+  #   serviceConfig = {
+  #     ExecStart = "${config.services.paperless.package}/bin/paperless-ngx document_exporter -na -nt -f -d ${stalwartBackupDir}";
+  #     ReadWritePaths = [
+  #       dataDir
+  #       config.services.idmail.dataDir
+  #       stalwartBackupDir
+  #     ];
+  #     Restart = "no";
+  #     Type = "oneshot";
+  #   };
+  #   inherit (cfg) environment;
+  #   requiredBy = ["restic-backups-storage-box-dusk.service"];
+  #   before = ["restic-backups-storage-box-dusk.service"];
+  # };
+  #
+  # # Needed so we don't run out of tmpfs space for large backups.
+  # # Technically this could be cleared each boot but whatever.
+  # environment.persistence."/state".directories = [
+  #   {
+  #     directory = stalwartBackupDir;
+  #     user = "stalwart-mail";
+  #     group = "stalwart-mail";
+  #     mode = "0700";
+  #   }
+  # ];
+  #
+  # backups.storageBoxes.dusk = {
+  #   subuser = "stalwart";
+  #   paths = [stalwartBackupDir];
+  # };
 }

pkgs/default.nix

@@ -4,6 +4,7 @@ _inputs: [
   (_final: prev: {
     deploy = prev.callPackage ./deploy.nix {};
     git-fuzzy = prev.callPackage ./git-fuzzy {};
+    stalwart-mail = prev.callPackage ./stal.nix {};
     kanidm = prev.kanidm.overrideAttrs (old: let
       provisionSrc = prev.fetchFromGitHub {
         owner = "oddlama";

pkgs/stal.nix

@@ -0,0 +1,149 @@
+{
+  lib,
+  rustPlatform,
+  fetchFromGitHub,
+  fetchpatch,
+  pkg-config,
+  protobuf,
+  bzip2,
+  openssl,
+  sqlite,
+  foundationdb,
+  zstd,
+  stdenv,
+  darwin,
+  nix-update-script,
+  nixosTests,
+  rocksdb_8_11,
+}: let
+  # Stalwart depends on rocksdb crate:
+  # https://github.com/stalwartlabs/mail-server/blob/v0.8.0/crates/store/Cargo.toml#L10
+  # which expects a specific rocksdb versions:
+  # https://github.com/rust-rocksdb/rust-rocksdb/blob/v0.22.0/librocksdb-sys/Cargo.toml#L3
+  # See upstream issue for rocksdb 9.X support
+  # https://github.com/stalwartlabs/mail-server/issues/407
+  rocksdb = rocksdb_8_11;
+  version = "0.9.0";
+in
+  rustPlatform.buildRustPackage {
+    pname = "stalwart-mail";
+    inherit version;
+
+    src = fetchFromGitHub {
+      owner = "stalwartlabs";
+      repo = "mail-server";
+      # XXX: We need to use a revisoin two commits after v0.9.0, which includes fixes for test cases.
+      # Can be reverted to "v${version}" next release.
+      rev = "2a12e251f2591b7785d7a921364f125d2e9c1e6e";
+      hash = "sha256-qoU09tLpOlsy5lKv2GdCV23bd70hnNZ0r/O5APGVDyw=";
+      fetchSubmodules = true;
+    };
+
+    cargoHash = "sha256-rGCu3J+hTxiIENDIQM/jPz1wUNJr0ouoa1IkwWKfOWM=";
+
+    patches = [
+      # Remove "PermissionsStartOnly" from systemd service files,
+      # which is deprecated and conflicts with our module's ExecPreStart.
+      # Upstream PR: https://github.com/stalwartlabs/mail-server/pull/528
+      (fetchpatch {
+        url = "https://github.com/stalwartlabs/mail-server/pull/528/commits/6e292b3d7994441e58e367b87967c9a277bce490.patch";
+        hash = "sha256-j/Li4bYNE7IppxG3FGfljra70/rHyhRvDgOkZOlhMHY=";
+      })
+    ];
+
+    nativeBuildInputs = [
+      pkg-config
+      protobuf
+      rustPlatform.bindgenHook
+    ];
+
+    buildInputs =
+      [
+        bzip2
+        openssl
+        sqlite
+        foundationdb
+        zstd
+      ]
+      ++ lib.optionals stdenv.isDarwin [
+        darwin.apple_sdk.frameworks.CoreFoundation
+        darwin.apple_sdk.frameworks.Security
+        darwin.apple_sdk.frameworks.SystemConfiguration
+      ];
+
+    env = {
+      OPENSSL_NO_VENDOR = true;
+      ZSTD_SYS_USE_PKG_CONFIG = true;
+      ROCKSDB_INCLUDE_DIR = "${rocksdb}/include";
+      ROCKSDB_LIB_DIR = "${rocksdb}/lib";
+    };
+
+    postInstall = ''
+      mkdir -p $out/etc/stalwart
+      cp resources/config/spamfilter.toml $out/etc/stalwart/spamfilter.toml
+      cp -r resources/config/spamfilter $out/etc/stalwart/
+
+      mkdir -p $out/lib/systemd/system
+
+      substitute resources/systemd/stalwart-mail.service $out/lib/systemd/system/stalwart-mail.service \
+        --replace "__PATH__" "$out"
+    '';
+
+    checkFlags = [
+      # Require running mysql, postgresql daemon
+      "--skip=directory::imap::imap_directory"
+      "--skip=directory::internal::internal_directory"
+      "--skip=directory::ldap::ldap_directory"
+      "--skip=directory::sql::sql_directory"
+      "--skip=store::blob::blob_tests"
+      "--skip=store::lookup::lookup_tests"
+      # thread 'directory::smtp::lmtp_directory' panicked at tests/src/store/mod.rs:122:44:
+      # called `Result::unwrap()` on an `Err` value: Os { code: 2, kind: NotFound, message: "No such file or directory" }
+      "--skip=directory::smtp::lmtp_directory"
+      # thread 'imap::imap_tests' panicked at tests/src/imap/mod.rs:436:14:
+      # Missing store type. Try running `STORE=<store_type> cargo test`: NotPresent
+      "--skip=imap::imap_tests"
+      # thread 'jmap::jmap_tests' panicked at tests/src/jmap/mod.rs:303:14:
+      # Missing store type. Try running `STORE=<store_type> cargo test`: NotPresent
+      "--skip=jmap::jmap_tests"
+      # Failed to read system DNS config: io error: No such file or directory (os error 2)
+      "--skip=smtp::inbound::data::data"
+      # Expected "X-My-Header: true" but got Received: from foobar.net (unknown [10.0.0.123])
+      "--skip=smtp::inbound::scripts::sieve_scripts"
+      # panicked at tests/src/smtp/outbound/smtp.rs:173:5:
+      "--skip=smtp::outbound::smtp::smtp_delivery"
+      # thread 'smtp::queue::retry::queue_retry' panicked at tests/src/smtp/queue/retry.rs:119:5:
+      # assertion `left == right` failed
+      #   left: [1, 2, 2]
+      #  right: [1, 2, 3]
+      "--skip=smtp::queue::retry::queue_retry"
+      # Missing store type. Try running `STORE=<store_type> cargo test`: NotPresent
+      "--skip=store::store_tests"
+      # thread 'config::parser::tests::toml_parse' panicked at crates/utils/src/config/parser.rs:463:58:
+      # called `Result::unwrap()` on an `Err` value: "Expected ['\\n'] but found '!' in value at line 70."
+      "--skip=config::parser::tests::toml_parse"
+      # error[E0432]: unresolved import `r2d2_sqlite`
+      # use of undeclared crate or module `r2d2_sqlite`
+      "--skip=backend::sqlite::pool::SqliteConnectionManager::with_init"
+      # thread 'smtp::reporting::analyze::report_analyze' panicked at tests/src/smtp/reporting/analyze.rs:88:5:
+      # assertion `left == right` failed
+      #   left: 0
+      #  right: 12
+      "--skip=smtp::reporting::analyze::report_analyze"
+    ];
+
+    doCheck = !(stdenv.isLinux && stdenv.isAarch64);
+
+    passthru = {
+      update-script = nix-update-script {};
+      tests.stalwart-mail = nixosTests.stalwart-mail;
+    };
+
+    meta = with lib; {
+      description = "Secure & Modern All-in-One Mail Server (IMAP, JMAP, SMTP)";
+      homepage = "https://github.com/stalwartlabs/mail-server";
+      changelog = "https://github.com/stalwartlabs/mail-server/blob/${version}/CHANGELOG";
+      license = licenses.agpl3Only;
+      maintainers = with maintainers; [happysalada onny oddlama];
+    };
+  }