[DB][Web] optimize qhandler by keeping SHA2 in new column qhash (#6556)

* [DB][Web] optimize qhandler by keeping SHA2(id+qid) in new column quarantine.qhash, for feature #6555, might also help with #6361

* rspamd: only add qhash to new entries while passing rspamd not all existing

* compose: bump dovecot image + push to registry

---------

Co-authored-by: DerLinkman <niklas.meyer@servercow.de>

data/Dockerfiles/dovecot/quarantine_notify.py

@@ -76,7 +76,7 @@ try:
 
   def notify_rcpt(rcpt, msg_count, quarantine_acl, category):
     if category == "add_header": category = "add header"
-    meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
+    meta_query = query_mysql('SELECT `qhash`, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
     print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count))
     if len(meta_query) == 0:
       return

data/conf/rspamd/meta_exporter/pipe.php

@@ -236,6 +236,9 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
       ':action' => $action,
       ':fuzzy_hashes' => $fuzzy
     ));
+    $lastId = $pdo->lastInsertId();
+    $stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
+    $stmt_update->execute(array(':id' => $lastId));
     $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
       SELECT `id`
       FROM (

data/web/inc/functions.quarantine.inc.php

@@ -22,7 +22,7 @@ function quarantine($_action, $_data = null) {
         return false;
       }
       $stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt`
-        WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash
+        WHERE `qhash` = :hash
           AND user_acl.quarantine = 1
           AND rcpt IN (SELECT username FROM mailbox)');
       $stmt->execute(array(':hash' => $hash));
@@ -65,7 +65,7 @@ function quarantine($_action, $_data = null) {
         return false;
       }
       $stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt`
-        WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash
+        WHERE `qhash` = :hash
           AND `user_acl`.`quarantine` = 1
           AND `username` IN (SELECT `username` FROM `mailbox`)');
       $stmt->execute(array(':hash' => $hash));
@@ -833,7 +833,7 @@ function quarantine($_action, $_data = null) {
         )));
         return false;
       }
-      $stmt = $pdo->prepare('SELECT * FROM `quarantine` WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash');
+      $stmt = $pdo->prepare('SELECT * FROM `quarantine` WHERE `qhash` = :hash');
       $stmt->execute(array(':hash' => $hash));
       return $stmt->fetch(PDO::FETCH_ASSOC);
     break;

data/web/inc/init_db.inc.php

@@ -4,7 +4,7 @@ function init_db_schema()
   try {
     global $pdo;
 
-    $db_version = "27012025_1555";
+    $db_version = "21052025_2215";
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -345,10 +345,14 @@ function init_db_schema()
           "notified" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
           "user" => "VARCHAR(255) NOT NULL DEFAULT 'unknown'",
+          "qhash" => "VARCHAR(255)",
         ),
         "keys" => array(
           "primary" => array(
             "" => array("id")
+          ),
+          "key" => array(
+            "qhash" => array("qhash")
           )
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
@@ -1482,6 +1486,10 @@ function init_db_schema()
         'msg' => 'db_init_complete'
       );
     }
+
+    // fill quarantine.qhash
+    $pdo->query("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE ISNULL(`qhash`)");
+
   } catch (PDOException $e) {
     if (php_sapi_name() == "cli") {
       echo "DB initialization failed: " . print_r($e, true) . PHP_EOL;