Various changes...

data/Dockerfiles/acme/Dockerfile

@@ -3,20 +3,21 @@ FROM alpine:3.6
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
 RUN apk add --update --no-cache \
-	bash \
-	curl \
-	openssl \
-	bind-tools \
-	jq \
+  bash \
+  curl \
+  openssl \
+  bind-tools \
+  jq \
   libressl-dev \
   libbsd-dev \
   libseccomp-dev \
-	mariadb-client \
+  mariadb-client \
   tini \
   make \
   gcc \
   libressl \
   libc-dev \
+  redis \
   linux-headers \
   ca-certificates \
   && curl -s https://kristaps.bsd.lv/acme-client/snapshots/acme-client-portable.tgz | tar xfvz - \

data/Dockerfiles/acme/docker-entrypoint.sh

@@ -2,16 +2,30 @@
 set -o pipefail
 exec 5>&1
 
+log_f() {
+  if [[ ${2} == "no_nl" ]]; then
+    echo -n "$(date) - ${1}"
+  elif [[ ${2} == "no_date" ]]; then
+    echo "${1}"
+  elif [[ ${2} != "redis_only" ]]; then
+    echo "$(date) - ${1}"
+  fi
+  redis-cli -h redis LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
+    tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
+  redis-cli -h redis LTRIM ACME_LOG 0 9999 > /dev/null
+}
+
 if [[ "${SKIP_LETS_ENCRYPT}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   log_f "SKIP_LETS_ENCRYPT=y, skipping Let's Encrypt..."
   sleep 365d
   exec $(readlink -f "$0")
 fi
 
-echo "Waiting for Docker API..."
+log_f "Waiting for Docker API..." no_nl
 until ping dockerapi -c1 > /dev/null; do
   sleep 1
 done
+log_f "Found Docker API" no_date
 
 ACME_BASE=/var/lib/acme
 SSL_EXAMPLE=/var/lib/ssl-example
@@ -20,21 +34,12 @@ mkdir -p ${ACME_BASE}/acme/private
 
 restart_containers(){
   for container in $*; do
-    echo "Restarting ${container}..."
-    curl -X POST http://dockerapi:8080/containers/${container}/restart
+    log_f "Restarting ${container}..." no_nl
+    C_REST_OUT=$(curl -X POST http://dockerapi:8080/containers/${container}/restart | jq -r '.msg')
+    log_f "${C_REST_OUT}" no_date
   done
 }
 
-log_f() {
-  if [[ ${2} == "no_nl" ]]; then
-    echo -n "$(date) - ${1}"
-  elif [[ ${2} == "no_date" ]]; then
-    echo "${1}"
-  else
-    echo "$(date) - ${1}"
-  fi
-}
-
 array_diff() {
   # https://stackoverflow.com/questions/2312762, Alex Offshore
   eval local ARR1=\(\"\${$2[@]}\"\)
@@ -123,16 +128,23 @@ while true; do
   declare -a VALIDATED_CONFIG_DOMAINS
   declare -a ADDITIONAL_VALIDATED_SAN
   IFS=',' read -r -a ADDITIONAL_SAN_ARR <<< "${ADDITIONAL_SAN}"
-  IPV4=$(get_ipv4)
+  until [[ ${IPV4} == ${EXTERNAL_IPV4} ]]; do
+    IPV4=$(get_ipv4)
+    if [[ ${IPV4} != ${EXTERNAL_IPV4} ]]; then
+      echo "Waiting for correct source ip..."
+      sleep 30s
+    fi
+  done
   # Container ids may have changed
   CONTAINERS_RESTART=($(curl --silent http://dockerapi:8080/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("nginx-mailcow") or contains("postfix-mailcow") or contains("dovecot-mailcow")) | .id' | tr "\n" " "))
 
-  log_f "Waiting for domain tables... " no_nl
+  log_f "Waiting for domain table... " no_nl
   while [[ -z ${DOMAIN_TABLE} ]]; do
+    curl --silent http://nginx/ >/dev/null 2>&1
     DOMAIN_TABLE=$(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'domain'" -Bs)
     [[ -z ${DOMAIN_TABLE} ]] && sleep 10
   done
-  log_f "OK" no_date
+  log_f "Found domain tables." no_date
 
   while read domains; do
     SQL_DOMAIN_ARR+=("${domains}")
@@ -226,6 +238,7 @@ while true; do
 
   case "$?" in
     0) # new certs
+      log_f "${ACME_RESPONSE}" redis_only
       # cp the new certificates and keys
       cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
       cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
@@ -239,6 +252,7 @@ while true; do
       restart_containers ${CONTAINERS_RESTART[*]}
       ;;
     1) # failure
+      log_f "${ACME_RESPONSE}" redis_only
       if [[ $ACME_RESPONSE =~ "No registration exists" ]]; then
         log_f "Registration keys are invalid, deleting old keys and restarting..."
         rm ${ACME_BASE}/acme/private/account.key
@@ -268,6 +282,7 @@ while true; do
       exec $(readlink -f "$0")
       ;;
     2) # no change
+      log_f "${ACME_RESPONSE}" redis_only
       if ! diff ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem; then
         log_f "Certificate was not changed, but active certificate does not match the verified certificate, fixing and restarting containers..."
         cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
@@ -280,9 +295,11 @@ while true; do
         cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
         TRIGGER_RESTART=1
       fi
+      log_f "Certificate was not changed"
       [[ ${TRIGGER_RESTART} == 1 ]] && restart_containers ${CONTAINERS_RESTART[*]}
       ;;
     *) # unspecified
+      log_f "${ACME_RESPONSE}" redis_only
       if [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ]]; then
         log_f "Error requesting certificate, restoring previous certificate from backup and restarting containers...."
         cp ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ${ACME_BASE}/cert.pem

data/Dockerfiles/dockerapi/server.py

@@ -6,6 +6,9 @@ from threading import Thread
 import docker
 import signal
 import time
+import os
+import re
+import sys
 
 docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
 app = Flask(__name__)
@@ -15,7 +18,7 @@ class containers_get(Resource):
   def get(self):
     containers = {}
     try:
-      for container in docker_client.containers.list(all=True):
+      for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME']}):
         containers.update({container.attrs['Id']: container.attrs})
       return containers
     except Exception as e:
@@ -25,19 +28,30 @@ class container_get(Resource):
   def get(self, container_id):
     if container_id and container_id.isalnum():
       try:
-        for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+        for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
           return container.attrs
       except Exception as e:
           return jsonify(type='danger', msg=e)
     else:
       return jsonify(type='danger', msg='no or invalid id defined')
 
+class container_logs(Resource):
+  def get(self, container_id, lines):
+    if container_id and container_id.isalnum() and lines:
+      try:
+        for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
+          return container.logs(stdout=True, stderr=True, stream=False, tail=lines)
+      except Exception as e:
+          return jsonify(type='danger', msg=e)
+    else:
+      return jsonify(type='danger', msg='no or invalid id defined')
+
 class container_post(Resource):
   def post(self, container_id, post_action):
     if container_id and container_id.isalnum() and post_action:
       if post_action == 'stop':
         try:
-          for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+          for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
             container.stop()
           return jsonify(type='success', msg='command completed successfully')
         except Exception as e:
@@ -45,7 +59,7 @@ class container_post(Resource):
 
       elif post_action == 'start':
         try:
-          for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+          for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
             container.start()
           return jsonify(type='success', msg='command completed successfully')
         except Exception as e:
@@ -53,7 +67,7 @@ class container_post(Resource):
 
       elif post_action == 'restart':
         try:
-          for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+          for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
             container.restart()
           return jsonify(type='success', msg='command completed successfully')
         except Exception as e:
@@ -66,16 +80,28 @@ class container_post(Resource):
 
         if request.json['cmd'] == 'sieve_list' and request.json['username']:
           try:
-            for container in docker_client.containers.list(filters={"id": container_id}):
+            for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
               return container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve list -u '" + request.json['username'].replace("'", "'\\''") + "'"], user='vmail')
           except Exception as e:
             return jsonify(type='danger', msg=e)
         elif request.json['cmd'] == 'sieve_print' and request.json['script_name'] and request.json['username']:
           try:
-            for container in docker_client.containers.list(filters={"id": container_id}):
+            for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
               return container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve get -u '" + request.json['username'].replace("'", "'\\''") + "' '" + request.json['script_name'].replace("'", "'\\''") + "'"], user='vmail')
           except Exception as e:
             return jsonify(type='danger', msg=e)
+        elif request.json['cmd'] == 'worker_password' and request.json['raw']:
+          try:
+            for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
+              hash = container.exec_run(["/bin/bash", "-c", "/usr/bin/rspamadm pw -e -p '" + request.json['raw'].replace("'", "'\\''") + "'"], user='_rspamd')
+              f = open("/access.inc", "w")
+              f.write('enable_password = "' + re.sub('[^0-9a-zA-Z\$]+', '', hash.rstrip()) + '";\n')
+              f.close()
+              container.restart()
+              return jsonify(type='success', msg='command completed successfully')
+          except Exception as e:
+            return jsonify(type='danger', msg=e)
+
         else:
           return jsonify(type='danger', msg='Unknown command')
 
@@ -99,6 +125,7 @@ def startFlaskAPI():
 
 api.add_resource(containers_get, '/containers/json')
 api.add_resource(container_get, '/containers/<string:container_id>/json')
+api.add_resource(container_logs, '/containers/<string:container_id>/logs/<int:lines>')
 api.add_resource(container_post, '/containers/<string:container_id>/<string:post_action>')
 
 if __name__ == '__main__':

data/Dockerfiles/phpfpm/Dockerfile

@@ -41,8 +41,8 @@ RUN apk add -U --no-cache libxml2-dev \
     Net_Sieve \
     NET_SMTP \
     Mail_mime \
-	&& pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} \
-	&& docker-php-ext-enable redis apcu memcached imagick \
+	&& pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse \
+	&& docker-php-ext-enable redis apcu memcached imagick mailparse \
 	&& pecl clear-cache \
 	&& docker-php-ext-configure intl \
   && docker-php-ext-install -j 4 intl gettext ldap sockets soap pdo pdo_mysql xmlrpc gd zip pcntl opcache \

data/Dockerfiles/postfix/whitelist_forwardinghosts.sh

@@ -6,7 +6,7 @@ while read QUERY; do
 		echo "500 dunno"
 		continue
 	fi
-	result=$(curl -s http://172.22.1.251:8081/forwardinghosts.php?host=${QUERY[1]})
+	result=$(curl -s http://nginx:8081/forwardinghosts.php?host=${QUERY[1]})
 	logger -t whitelist_forwardinghosts -p mail.info "Look up ${QUERY[1]} on whitelist, result $result"
 	echo ${result}
 done

data/Dockerfiles/watchdog/watchdog.sh

@@ -28,21 +28,19 @@ progress() {
   [[ ${CURRENT} -gt ${TOTAL} ]] && return
   [[ ${CURRENT} -lt 0 ]] && CURRENT=0
   PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
-  log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}"
-  log_data "$(printf "%d,%d,%d,%d" ${PERCENT} ${CURRENT} ${TOTAL} ${DIFF})" "${SERVICE}"
+  redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
+  log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
 }
 
 log_msg() {
-	redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}" > /dev/null
+  if [[ ${2} != "no_redis" ]]; then
+    redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
+      tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
+  fi
+  redis-cli -h redis LTRIM WATCHDOG_LOG 0 9999 > /dev/null
   echo $(date) $(printf '%s\n' "${1}")
 }
 
-log_data() {
-  [[ -z ${1} ]] && return 1
-  [[ -z ${2} ]] && return 2
-	redis-cli -h redis LPUSH WATCHDOG_DATA "{\"time\":\"$(date +%s)\",\"service\":\"data\",\"$(printf '%s' "${2}")\":\"$(printf '%s' "${1}")\"}" > /dev/null
-}
-
 function mail_error() {
   [[ -z ${1} ]] && return 1
   [[ -z ${2} ]] && return 2
@@ -234,27 +232,6 @@ Empty
   return 1
 }
 
-dns_checks() {
-  err_count=0
-  diff_c=0
-  THRESHOLD=28
-  # Reduce error count by 2 after restarting an unhealthy container
-  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
-  while [ ${err_count} -lt ${THRESHOLD} ]; do
-    host_ip=$(get_container_ip unbound-mailcow)
-    err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_dns -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2)))
-    /usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2)))
-    dig +dnssec org. @${host_ip} | grep -E 'flags:.+ad' 1>&2; err_count=$(( ${err_count} + ($? * 2)))
-    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
-    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
-    progress "Unbound" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
-    sleep $(( ( RANDOM % 30 )  + 10 ))
-  done
-  return 1
-}
-
 # Create watchdog agents
 (
 while true; do
@@ -322,17 +299,6 @@ done
 ) &
 BACKGROUND_TASKS+=($!)
 
-(
-while true; do
-  if ! dns_checks; then
-    log_msg "Unbound hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "unbound-mailcow"
-    #echo unbound-mailcow > /tmp/com_pipe
-  fi
-done
-) &
-BACKGROUND_TASKS+=($!)
-
 (
 while true; do
   if ! rspamd_checks; then