[Rspamd] Apply ratelimit against authenticated user instead of envelope from

[PHP-FPM] Create PHP-FPM listeners 9001 (system) and 9002 (web), drop 9000
[Rspamd] Parse quarantine messages as utf8
[Rspamd] Use new schema for Rspamd bayes hashes and expire them in Redis
[SOGo] Change default logo
[SOGo] Use different keyserver by default in Dockerfile
[Rspamd] Add bad ASN list (disabled by default)
[Watchdog] Change the way we check PHP-FPM, change SOGo check
[Nginx] Change ports according to new PHP-FPM listeners
[Update] Fix PHP-FPM ports for existing non-mailcow Nginx sites

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,84 +1,78 @@
-FROM php:7.1-fpm-alpine
+FROM php:7.2-fpm-alpine3.7
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
-ENV REDIS_PECL 3.1.6
-ENV MEMCACHED_PECL 3.0.4
-ENV APCU_PECL 5.1.9
+ENV APCU_PECL 5.1.10
 ENV IMAGICK_PECL 3.4.3
 ENV MAILPARSE_PECL 3.0.2
+ENV MEMCACHED_PECL 3.0.4
+ENV REDIS_PECL 3.1.6
 
-RUN apk add -U --no-cache libxml2-dev \
-	icu-dev \
-	imap-dev \
-	libmemcached-dev \
-	cyrus-sasl-dev \
-	pcre-dev \
-	icu-libs \
-	redis \
-	mysql-client \
-	bash \
-	autoconf \
-	g++ \
-	make \
-	openssl \
-	openssl-dev \
-	samba-client \
-	libpng \
-	libpng-dev \
-	libjpeg-turbo-dev \
-	libwebp-dev \
-	zlib-dev \
-	libxpm-dev \
-	c-client \
-  imagemagick-dev \
-  imagemagick \
-  libtool \
+RUN apk add -U --no-cache autoconf \
+  bash \
+  c-client \
+  cyrus-sasl-dev \
   freetype \
-  libpng \
-  libjpeg-turbo \
   freetype-dev \
-  libpng-dev \
-  libjpeg-turbo-dev\
+  g++ \
   gettext-dev \
-  openldap-dev \
+  icu-dev \
+  icu-libs \
+  imagemagick \
+  imagemagick-dev \
+  imap-dev \
+  libjpeg-turbo \
+  libjpeg-turbo-dev \
+  libmemcached-dev \
+  libpng \
+  libpng-dev \
+  libressl \
+  libressl-dev \
   librsvg \
+  libtool \
+  libwebp-dev \
+  libxml2-dev \
+  libxpm-dev \
+  make \
+  mysql-client \
+  openldap-dev \
+  pcre-dev \
+  redis \
+  samba-client \
+  zlib-dev \
   && pear install channel://pear.php.net/Net_IDNA2-0.2.0 \
     channel://pear.php.net/Auth_SASL-1.1.0 \
     Net_IMAP \
     Net_Sieve \
     NET_SMTP \
     Mail_mime \
-	&& pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse-${MAILPARSE_PECL} \
-	&& docker-php-ext-enable redis apcu memcached imagick mailparse \
-	&& pecl clear-cache \
-	&& docker-php-ext-configure intl \
+  && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse-${MAILPARSE_PECL} \
+  && docker-php-ext-enable apcu imagick mailparse memcached redis \
+  && pecl clear-cache \
+  && docker-php-ext-configure intl \
   && docker-php-ext-configure gd \
-      --with-gd \
-      --enable-gd-native-ttf \
-      --with-freetype-dir=/usr/include/ \
-      --with-png-dir=/usr/include/ \
-      --with-jpeg-dir=/usr/include/ \
-  && docker-php-ext-install -j 4 intl gettext ldap sockets soap pdo pdo_mysql xmlrpc gd zip pcntl opcache \
+    --with-gd \
+    --enable-gd-native-ttf \
+    --with-freetype-dir=/usr/include/ \
+    --with-png-dir=/usr/include/ \
+    --with-jpeg-dir=/usr/include/ \
+  && docker-php-ext-install -j 4 gd gettext intl ldap opcache pcntl pdo pdo_mysql soap sockets xmlrpc zip \
   && docker-php-ext-configure imap --with-imap --with-imap-ssl \
-	&& docker-php-ext-install -j 4 imap \
-	&& apk del --purge autoconf \
-  g++ \
-  make \
-  libxml2-dev \
-  icu-dev \
-  imap-dev \
-  openssl-dev \
-  cyrus-sasl-dev \
-  pcre-dev \
-  libpng-dev \
-  libpng-dev \
-  libjpeg-turbo-dev \
-  libwebp-dev \
-  zlib-dev \
-  imagemagick-dev \
-  freetype-dev \
-  libpng-dev \
-  libjpeg-turbo-dev
+  && docker-php-ext-install -j 4 imap \
+  && apk del --purge autoconf \
+    cyrus-sasl-dev \
+    freetype-dev \
+    g++ \
+    icu-dev \
+    imagemagick-dev \
+    imap-dev \
+    libjpeg-turbo-dev \
+    libpng-dev \
+    libressl-dev \
+    libwebp-dev \
+    libxml2-dev \
+    make \
+    pcre-dev \
+    zlib-dev
 
 COPY ./docker-entrypoint.sh /
 

data/Dockerfiles/sogo/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN mkdir /usr/share/doc/sogo \
 	&& touch /usr/share/doc/sogo/empty.sh \
-	&& apt-key adv --keyserver keys.gnupg.net --recv-key 0x810273C4 \
+	&& apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \
 	&& echo "deb http://packages.inverse.ca/SOGo/nightly/4/debian/ stretch stretch" > /etc/apt/sources.list.d/sogo.list \
 	&& apt-get update && apt-get install -y --force-yes \
 		sogo \

data/Dockerfiles/sogo/theme-blue.js

@@ -16,16 +16,16 @@
      * Define the Alternative theme
      */
     $mdThemingProvider.theme('mailcow')
-      .primaryPalette('indigo', {
+      .primaryPalette('blue', {
         'default': '700',  // top toolbar
-        'hue-1': '400',
-        'hue-2': '600',    // sidebar toolbar
+        'hue-1': '500',
+        'hue-2': '700',    // sidebar toolbar
         'hue-3': 'A700'
       })
-      .accentPalette('indigo', {
-        'default': '500',  // fab buttons
+      .accentPalette('blue', {
+        'default': '700',  // fab buttons
         'hue-1': '50',     // center list toolbar
-        'hue-2': '400',
+        'hue-2': '600',
         'hue-3': 'A700'
       })
       .backgroundPalette('grey', {
@@ -35,16 +35,16 @@
         'hue-3': '300'
       });
     $mdThemingProvider.theme('default')
-      .primaryPalette('indigo', {
+      .primaryPalette('blue', {
         'default': '700',  // top toolbar
-        'hue-1': '400',
-        'hue-2': '600',    // sidebar toolbar
+        'hue-1': '500',
+        'hue-2': '700',    // sidebar toolbar
         'hue-3': 'A700'
       })
-      .accentPalette('indigo', {
-        'default': '500',  // fab buttons
+      .accentPalette('blue', {
+        'default': '700',  // fab buttons
         'hue-1': '50',     // center list toolbar
-        'hue-2': '400',
+        'hue-2': '600',
         'hue-3': 'A700'
       })
       .backgroundPalette('grey', {

data/Dockerfiles/watchdog/watchdog.sh

@@ -128,7 +128,7 @@ sogo_checks() {
     host_ip=$(get_container_ip sogo-mailcow)
     err_c_cur=${err_count}
     /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /WebServerResources/css/theme-default.css -p 9192 -R md-default-theme 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\sGroupware" 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -190,8 +190,8 @@ phpfpm_checks() {
   while [ ${err_count} -lt ${THRESHOLD} ]; do
     host_ip=$(get_container_ip php-fpm-mailcow)
     err_c_cur=${err_count}
-    cgi-fcgi -bind -connect ${host_ip}:9000 | grep "Content-type" 1>&2; err_count=$(( ${err_count} + ($? * 2)))
-    cgi-fcgi -bind -connect ${host_ip}:9001 | grep "Content-type" 1>&2; err_count=$(( ${err_count} + ($? * 2)))
+    nc -z ${host_ip} 9001 ; err_count=$(( ${err_count} + ($? * 2)))
+    nc -z ${host_ip} 9002 ; err_count=$(( ${err_count} + ($? * 2)))
     /usr/lib/nagios/plugins/check_ping -4 -H ${host_ip} -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))

data/assets/nextcloud/nextcloud.conf

@@ -82,10 +82,13 @@ server {
     #Avoid sending the security headers twice
     fastcgi_param modHeadersAvailable true;
     fastcgi_param front_controller_active true;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9002;
     fastcgi_intercept_errors on;
     fastcgi_request_buffering off;
-    client_max_body_size 10G;
+    client_max_body_size 0;
+    fastcgi_param PHP_VALUE "max_execution_time = 1200
+                             max_input_time = 1200";
+    fastcgi_read_timeout 1200;
   }
 
   location ~ ^/(?:updater|ocs-provider)(?:$|/) {

data/assets/nextcloud/site.nextcloud.custom

@@ -16,10 +16,13 @@
       fastcgi_param HTTPS on;
       fastcgi_param modHeadersAvailable true;
       fastcgi_param front_controller_active true;
-      fastcgi_pass phpfpm:9000;
+      fastcgi_pass phpfpm:9002;
       fastcgi_intercept_errors on;
       fastcgi_request_buffering off;
-      client_max_body_size 10G;
+      client_max_body_size 0;
+      fastcgi_param PHP_VALUE "max_execution_time = 1200
+                               max_input_time = 1200";
+      fastcgi_read_timeout 1200;
     }
     location ~ ^/nextcloud/(?:updater|ocs-provider)(?:$|/) {
       try_files $uri/ =404;

data/conf/nginx/meta_exporter.conf

@@ -10,7 +10,7 @@ server {
     client_max_body_size 10M;
     try_files $uri =404;
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9001;
     fastcgi_index pipe.php;
     include fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;

data/conf/nginx/site.conf

@@ -35,7 +35,6 @@ server {
   add_header X-XSS-Protection "1; mode=block";
   add_header X-Robots-Tag none;
   add_header X-Download-Options noopen;
-  add_header X-Frame-Options "SAMEORIGIN";
   add_header X-Permitted-Cross-Domain-Policies none;
 
   index index.php index.html;
@@ -76,7 +75,7 @@ server {
   location ~ \.php$ {
     try_files $uri =404;
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9002;
     fastcgi_index index.php;
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
@@ -96,9 +95,22 @@ server {
     expires $expires;
   }
 
+  location /lists/ {
+    proxy_pass       http://mm-web:8000/;
+    proxy_set_header Host      $http_host;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_redirect off;
+    expires $expires;
+  }
+
+  location /mm_static {
+    alias /opt/mm_web-data/static;
+  }
+
   location ~* ^/Autodiscover/Autodiscover.xml {
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9002;
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     try_files /autodiscover.php =404;
@@ -106,7 +118,7 @@ server {
 
   location ~* ^/Autodiscover/Autodiscover.json {
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9002;
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     try_files /autodiscover-json.php =404;
@@ -114,7 +126,7 @@ server {
 
   location ~ /(?:m|M)ail/(?:c|C)onfig-v1.1.xml {
     fastcgi_split_path_info ^(.+\.php)(/.+)$;
-    fastcgi_pass phpfpm:9000;
+    fastcgi_pass phpfpm:9002;
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     try_files /autoconfig.php =404;

data/conf/phpfpm/php-fpm.d/pools.conf

@@ -0,0 +1,25 @@
+[system-worker]
+user = www-data
+group = www-data
+pm = dynamic
+pm.max_children = 15
+pm.start_servers = 2
+pm.min_spare_servers = 2
+pm.max_spare_servers = 4
+listen = [::]:9001
+access.log = /proc/self/fd/2
+clear_env = no
+catch_workers_output = yes
+
+[web-worker]
+user = www-data
+group = www-data
+pm = dynamic
+pm.max_children = 50
+pm.start_servers = 2
+pm.min_spare_servers = 2
+pm.max_spare_servers = 4
+listen = [::]:9002
+access.log = /proc/self/fd/2
+clear_env = no
+catch_workers_output = yes

data/conf/rspamd/custom/bad_asn.map

@@ -0,0 +1,27 @@
+# High spam networks, disabled by default
+#201942:5 #Soltia Consulting SL - ipinfo.io
+#16276:5 #OVH
+#12876:5 #ONLINE S.A.S
+#31034:5
+#12874:5
+#30823:5
+#197071:5
+#42831:5 #UK Dedicated Servers Ltd
+#29119:5 #Aire Networks del Mediterraneo S.L.U.
+#13335:5 #Cloudflare
+#28753:5 #Leaseweb
+#61272:5 #Informacines sistemos ir technologijos
+#53755:5 #Input Output Flood LLC
+#29422:5 #FICIX Helsinki
+#62255:4 #Asmunda New Media Ltd
+#14061:4 #Digitalocean
+#55293:4 #A2 Hosting
+#63018:4 #US Dedicated
+#197518:2
+#44493:2
+#46606:2
+#49505:2
+#21100:2
+#197695:2
+#198068:2
+#43146:2

data/conf/rspamd/local.d/multimap.conf

@@ -27,6 +27,14 @@ KEEP_SPAM {
   action = "accept";
 }
 
+LOCAL_BL_ASN {
+  require_symbols = "!MAILCOW_WHITE";
+  type = "asn";
+  map = "$LOCAL_CONFDIR/custom/bad_asn.map";
+  score = 5;
+  description = "Sender's ASN is on the local blacklist";
+}
+
 #SPOOFED_SENDER {
 #  type = "rcpt";
 #  filter = "email:domain:tld";

data/conf/rspamd/local.d/statistic.conf

@@ -8,6 +8,8 @@ classifier "bayes" {
     min_tokens = 11;
     min_learns = 20;
     autolearn = [-20, 50];
+    new_schema = true;
+    expiry = 50d;
     per_user = <<EOD
 return function(task)
     local rcpt = task:get_recipients(1)

data/conf/rspamd/lua/rspamd.local.lua

@@ -59,16 +59,16 @@ rspamd_config:register_symbol({
     local redis_params = rspamd_parse_redis_server('dyn_rl')
     local rspamd_logger = require "rspamd_logger"
     local envfrom = task:get_from(1)
-    if not envfrom then
+    local uname = task:get_user():lower()
+    if not envfrom or not uname then
       return false
     end
     local env_from_domain = envfrom[1].domain:lower() -- get smtp from domain in lower case
-    local env_from_addr = envfrom[1].addr:lower() -- get smtp from addr in lower case
 
     local function redis_cb_user(err, data)
 
       if err or type(data) ~= 'string' then
-        rspamd_logger.infox(rspamd_config, "dynamic ratelimit request for user %s returned invalid or empty data (\"%s\") or error (\"%s\") - trying dynamic ratelimit for domain...", env_from_addr, data, err)
+        rspamd_logger.infox(rspamd_config, "dynamic ratelimit request for user %s returned invalid or empty data (\"%s\") or error (\"%s\") - trying dynamic ratelimit for domain...", uname, data, err)
 
         local function redis_key_cb_domain(err, data)
           if err or type(data) ~= 'string' then
@@ -91,7 +91,7 @@ rspamd_config:register_symbol({
           rspamd_logger.infox(rspamd_config, "cannot make request to load ratelimit for domain")
         end
       else
-        rspamd_logger.infox(rspamd_config, "found dynamic ratelimit in redis for user %s with value %s", env_from_addr, data)
+        rspamd_logger.infox(rspamd_config, "found dynamic ratelimit in redis for user %s with value %s", uname, data)
         task:insert_result('DYN_RL', 0.0, data)
       end
 
@@ -99,11 +99,11 @@ rspamd_config:register_symbol({
 
     local redis_ret_user = rspamd_redis_make_request(task,
       redis_params, -- connect params
-      env_from_addr, -- hash key
+      uname, -- hash key
       false, -- is write
       redis_cb_user, --callback
       'HGET', -- command
-      {'RL_VALUE', env_from_addr} -- arguments
+      {'RL_VALUE', uname} -- arguments
     )
     if not redis_ret_user then
       rspamd_logger.infox(rspamd_config, "cannot make request to load ratelimit for user")

data/conf/rspamd/meta_exporter/pipe.php

@@ -44,7 +44,8 @@ if (!function_exists('getallheaders'))  {
   }
 }
 
-$raw_data = file_get_contents('php://input');
+$raw_data_content = file_get_contents('php://input');
+$raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
 $headers = getallheaders();
 
 $qid      = $headers['X-Rspamd-Qid'];