dovecot: add Flatcurve FTS Engine as EXPERIMENTAL (#5920)

* dovecot: experimental added flatcurve backend + switch

* dovecot: bump docker image

data/Dockerfiles/dovecot/Dockerfile

@@ -24,6 +24,7 @@ RUN addgroup -g 5000 vmail \
   envsubst \
   ca-certificates \
   curl \
+  coreutils \
   jq \
   lua \
   lua-cjson \
@@ -104,6 +105,7 @@ RUN addgroup -g 5000 vmail \
   dovecot-pigeonhole-plugin \
   dovecot-pop3d \
   dovecot-fts-solr \
+  dovecot-fts-flatcurve \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
   && chmod +x /usr/local/bin/gosu \
@@ -127,6 +129,7 @@ COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py
 COPY quota_notify.py /usr/local/bin/quota_notify.py
 COPY repl_health.sh /usr/local/bin/repl_health.sh
+COPY optimize-fts.sh /usr/local/bin/optimize-fts.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -29,6 +29,7 @@ ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
 # Create missing directories
 [[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/
 [[ ! -d /etc/dovecot/lua/ ]] && mkdir -p /etc/dovecot/lua/
+[[ ! -d /etc/dovecot/conf.d/ ]] && mkdir -p /etc/dovecot/conf.d/
 [[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
 [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
 [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo
@@ -109,7 +110,14 @@ EOF
 
 echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
-if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then
+echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m"
+echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m"
+echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m"
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
+elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
 echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
@@ -239,6 +247,47 @@ function script_deinit()
 end
 EOF
 
+# Temporarily set FTS depending on user choice inside mailcow.conf. Will be removed as soon as Solr is dropped
+if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+cat <<EOF > /etc/dovecot/conf.d/fts.conf
+# Autogenerated by mailcow
+plugin {
+    fts_autoindex = yes
+    fts_autoindex_exclude = \Junk
+    fts_autoindex_exclude2 = \Trash
+    fts = flatcurve
+
+    # These are not flatcurve settings, but required for Dovecot FTS. See
+    # Dovecot FTS Configuration link above for further information.
+    fts_languages = en es de
+    fts_tokenizer_generic = algorithm=simple
+    fts_tokenizers = generic email-address
+
+    # OPTIONAL: Recommended default FTS core configuration
+    fts_filters = normalizer-icu snowball stopwords
+    fts_filters_en = lowercase snowball english-possessive stopwords
+}
+EOF
+elif [[ ! "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+cat <<EOF > /etc/dovecot/conf.d/fts.conf
+# Autogenerated by mailcow
+plugin {
+  fts = solr
+  fts_autoindex = yes
+  fts_autoindex_exclude = \Junk
+  fts_autoindex_exclude2 = \Trash
+  fts_solr = url=http://solr:8983/solr/dovecot-fts/
+
+  fts_tokenizers = generic email-address
+  fts_tokenizer_generic = algorithm=simple
+
+  fts_filters = normalizer-icu snowball stopwords
+  fts_filters_en = lowercase snowball english-possessive stopwords
+}
+EOF
+fi
+
+
 # Replace patterns in app-passdb.lua
 sed -i "s/__DBUSER__/${DBUSER}/g" /etc/dovecot/lua/passwd-verify.lua
 sed -i "s/__DBPASS__/${DBPASS}/g" /etc/dovecot/lua/passwd-verify.lua
@@ -343,7 +392,6 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
 EOF
 fi
 
-
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem
@@ -387,7 +435,8 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/local/bin/maildir_gc.sh \
   /usr/local/sbin/stop-supervisor.sh \
   /usr/local/bin/quota_notify.py \
-  /usr/local/bin/repl_health.sh
+  /usr/local/bin/repl_health.sh \
+  /usr/local/bin/optimize-fts.sh
 
 # Prepare environment file for cronjobs
 printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh

data/Dockerfiles/dovecot/optimize-fts.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ && ! "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    exit 0
+else
+    doveadm fts optimize -A
+fi

data/Dockerfiles/solr/solr.sh

@@ -1,7 +1,15 @@
 #!/bin/bash
 
-if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  echo "FLATCURVE_EXPERIMENTAL=y, skipping Solr but enabling Flatcurve as FTS for Dovecot!"
+  echo "Solr will be removed in the future!"
+  sleep 365d
+  exit 0
+elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   echo "SKIP_SOLR=y, skipping Solr..."
+  echo "HINT: You could try the newer FTS Backend Flatcurve, which is currently in experimental state..."
+  echo "Simply set FLATCURVE_EXPERIMENTAL=y inside your mailcow.conf and restart the stack afterwards!"
+  echo "Solr will be removed in the future!"
   sleep 365d
   exit 0
 fi
@@ -57,5 +65,11 @@ if [[ "${1}" == "--bootstrap" ]]; then
   exit 0
 fi
 
+echo "Starting up Solr..."
+echo -e "\e[31mSolr is deprecated! You can try the new FTS System now by enabling FLATCURVE_EXPERIMENTAL=y inside mailcow.conf and restarting the stack\e[0m"
+echo -e "\e[31mSolr will be removed completely soon!\e[0m"
+
+sleep 15
+
 exec gosu solr solr-foreground
 

data/conf/dovecot/dovecot.conf

@@ -10,6 +10,7 @@
 auth_mechanisms = plain login
 #mail_debug = yes
 #auth_debug = yes
+#log_debug = category=fts-flatcurve # Activate Logging for Flatcurve FTS Searchings
 log_path = syslog
 disable_plaintext_auth = yes
 # Uncomment on NFS share
@@ -194,9 +195,6 @@ plugin {
   acl_shared_dict = file:/var/vmail/shared-mailboxes.db
   acl = vfile
   acl_user = %u
-  fts = solr
-  fts_autoindex = yes
-  fts_solr = url=http://solr:8983/solr/dovecot-fts/
   quota = dict:Userquota::proxy::sqlquota
   quota_rule2 = Trash:storage=+100%%
   sieve = /var/vmail/sieve/%u.sieve
@@ -305,6 +303,7 @@ replication_dsync_parameters = -d -l 30 -U -n INBOX
 !include_try /etc/dovecot/extra.conf
 !include_try /etc/dovecot/sogo-sso.conf
 !include_try /etc/dovecot/shared_namespace.conf
+!include_try /etc/dovecot/conf.d/fts.conf
 # </Includes>
 default_client_limit = 10400
 default_vsz_limit = 1024 M

docker-compose.yml

@@ -221,7 +221,7 @@ services:
             - sogo
 
     dovecot-mailcow:
-      image: mailcow/dovecot:1.29
+      image: mailcow/dovecot:1.30
       depends_on:
         - mysql-mailcow
         - netfilter-mailcow
@@ -264,6 +264,7 @@ services:
         - REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-}
         - REDIS_SLAVEOF_PORT=${REDIS_SLAVEOF_PORT:-}
         - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
+        - FLATCURVE_EXPERIMENTAL=${FLATCURVE_EXPERIMENTAL:-n}
       ports:
         - "${DOVEADM_PORT:-127.0.0.1:19991}:12345"
         - "${IMAP_PORT:-143}:143"
@@ -289,7 +290,7 @@ services:
         ofelia.job-exec.dovecot_sarules.schedule: "@every 24h"
         ofelia.job-exec.dovecot_sarules.command: "/bin/bash -c \"/usr/local/bin/sa-rules.sh\""
         ofelia.job-exec.dovecot_fts.schedule: "@every 24h"
-        ofelia.job-exec.dovecot_fts.command: "/usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true"
+        ofelia.job-exec.dovecot_fts.command: "/bin/bash -c \"/usr/local/bin/gosu vmail /usr/local/bin/optimize-fts.sh\""
         ofelia.job-exec.dovecot_repl_health.schedule: "@every 5m"
         ofelia.job-exec.dovecot_repl_health.command: "/bin/bash -c \"/usr/local/bin/gosu vmail /usr/local/bin/repl_health.sh\""
       ulimits:
@@ -553,7 +554,7 @@ services:
     
     ##### Will be removed soon #####
     solr-mailcow:
-      image: mailcow/solr:1.8.2
+      image: mailcow/solr:1.8.3
       restart: always
       depends_on:
         - netfilter-mailcow
@@ -565,6 +566,7 @@ services:
         - TZ=${TZ}
         - SOLR_HEAP=${SOLR_HEAP:-1024}
         - SKIP_SOLR=${SKIP_SOLR:-y}
+        - FLATCURVE_EXPERIMENTAL=${FLATCURVE_EXPERIMENTAL:-n}
       networks:
         mailcow-network:
           aliases: