[SQL Upgrade] Fix process of SQL upgrade, wait for command to complete and do not use pipe communication model

data/Dockerfiles/dockerapi/server.py

@@ -216,18 +216,19 @@ class container_post(Resource):
   # api call: container_post - post_action: exec - cmd: system - task: mysql_upgrade
   def container_post__exec__system__mysql_upgrade(self, container_id):
     for container in docker_client.containers.list(filters={"id": container_id}):
-      cmd = "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"
-      cmd_response = exec_cmd_container(container, cmd, user='mysql')
-
-      matched = False
-      for line in cmd_response.split("\n"):
-        if 'is already upgraded to' in line:
-          matched = True
-      if matched:
-        return jsonify(type='success', msg='mysql_upgrade: already upgraded')
+      sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"], user='mysql')
+      if sql_return.exit_code == 0:
+        matched = False
+        for line in sql_return.output.decode('utf-8').split("\n"):
+          if 'is already upgraded to' in line:
+            matched = True
+        if matched:
+          return jsonify(type='success', msg='mysql_upgrade: already upgraded', text=sql_return.output.decode('utf-8'))
+        else:
+          container.restart()
+          return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied', text=sql_return.output.decode('utf-8'))
       else:
-        container.restart()
-        return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied')
+        return jsonify(type='error', msg='mysql_upgrade: error running command')
 
 
   # api call: container_post - post_action: exec - cmd: reload - task: dovecot

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -28,50 +28,60 @@ fi
 # Check of mysql_upgrade
 
 CONTAINER_ID=
-# Todo: Better check if upgrade failed
-# This can happen due to a broken sogo_view
-[ -s /mysql_upgrade_loop ] && SQL_LOOP_C=$(cat /mysql_upgrade_loop)
-[ -z ${SQL_LOOP_C} ] && SQL_LOOP_C=0
 until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do
   CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" 2> /dev/null | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | .id" 2> /dev/null)
 done
 echo "MySQL @ ${CONTAINER_ID}"
-SQL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json' | jq -r .type)
-if [[ ${SQL_UPGRADE_RETURN} == 'warning' ]]; then
-  sleep 2
-  if [ ${SQL_LOOP_C} -lt 2 ]; then
-    echo $((SQL_LOOP_C+1)) > /mysql_upgrade_loop
-    echo "MySQL applied an upgrade"
-    POSTFIX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("postfix-mailcow")) | .id' | tr "\n" " "))
-    if [[ -z ${POSTFIX} ]]; then
-      echo "Could not determine Postfix container ID, skipping Postfix restart."
-    else
-      echo "Restarting Postfix"
-      curl -X POST --silent --insecure https://dockerapi/containers/${POSTFIX}/restart | jq -r '.msg'
-      echo "Sleeping 10 seconds..."
-      sleep 10
-    fi
-    echo "Restarting PHP-FPM, bye"
-    exit 1
-  else
-    rm /mysql_upgrade_loop
-    echo "MySQL upgrade was not applied previously, skipping. Restart php-fpm-mailcow to retry or run mysql_upgrade manually."
+SQL_LOOP_C=0
+SQL_CHANGED=0
+until [[ ${SQL_UPGRADE_STATUS} == 'success' ]]; do
+  if [ ${SQL_LOOP_C} -gt 4 ]; then
+    echo "Tried to upgrade MySQL and failed, giving up after ${SQL_LOOP_C} retries and starting container (oops, not good)"
+    break
+  fi
+  SQL_FULL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json')
+  SQL_UPGRADE_STATUS=$(echo ${SQL_FULL_UPGRADE_RETURN} | jq -r .type)
+  SQL_LOOP_C=$((SQL_LOOP_C+1))
+  echo "SQL upgrade iteration #${SQL_LOOP_C}"
+  if [[ ${SQL_UPGRADE_STATUS} == 'warning' ]]; then
+    SQL_CHANGED=1
+    echo "MySQL applied an upgrade, debug output:"
+    echo ${SQL_FULL_UPGRADE_RETURN}
+    sleep 3
     while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
-      echo "Waiting for SQL to return..."
+      echo "Waiting for SQL to return, please wait"
       sleep 2
     done
+    continue
+  elif [[ ${SQL_UPGRADE_STATUS} == 'success' ]]; then
+    echo "MySQL is up-to-date - debug output:"
+    echo ${SQL_FULL_UPGRADE_RETURN}
+  else
+    echo "No valid reponse for mysql_upgrade was received"
+  fi
+done
+
+# doing post-installation stuff, if SQL was upgraded
+if [ ${SQL_CHANGED} -eq 1 ]; then
+  POSTFIX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("postfix-mailcow")) | .id' | tr "\n" " "))
+  if [[ -z ${POSTFIX} ]]; then
+    echo "Could not determine Postfix container ID, skipping Postfix restart."
+  else
+    echo "Restarting Postfix"
+    curl -X POST --silent --insecure https://dockerapi/containers/${POSTFIX}/restart | jq -r '.msg'
+    echo "Sleeping 5 seconds..."
+    sleep 5
   fi
-else
-  echo "MySQL is up-to-date"
 fi
 
 # Trigger db init
 echo "Running DB init..."
 php -c /usr/local/etc/php -f /web/inc/init_db.inc.php
 
-# Migrate domain map
+# Recreating domain map
+echo "Rebuilding domain map in Redis..."
 declare -a DOMAIN_ARR
-redis-cli -h redis-mailcow DEL DOMAIN_MAP
+  redis-cli -h redis-mailcow DEL DOMAIN_MAP > /dev/null
 while read line
 do
   DOMAIN_ARR+=("$line")
@@ -83,7 +93,7 @@ done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${D
 
 if [[ ! -z ${DOMAIN_ARR} ]]; then
 for domain in "${DOMAIN_ARR[@]}"; do
-  redis-cli -h redis-mailcow HSET DOMAIN_MAP ${domain} 1
+  redis-cli -h redis-mailcow HSET DOMAIN_MAP ${domain} 1 > /dev/null
 done
 fi
 

docker-compose.yml

@@ -95,7 +95,7 @@ services:
             - rspamd
 
     php-fpm-mailcow:
-      image: mailcow/phpfpm:1.45
+      image: mailcow/phpfpm:1.46
       build: ./data/Dockerfiles/phpfpm
       command: "php-fpm -d date.timezone=${TZ} -d expose_php=0"
       depends_on:
@@ -394,7 +394,7 @@ services:
             - watchdog
 
     dockerapi-mailcow:
-      image: mailcow/dockerapi:1.32
+      image: mailcow/dockerapi:1.33
       restart: always
       build: ./data/Dockerfiles/dockerapi
       oom_kill_disable: true