feat: handle event propagation via DB (HA)

config.sample.yml

@@ -125,6 +125,15 @@ uploads:
 
 offline: false
 
+# ---------------------------------------------------------------------
+# High-Availability
+# ---------------------------------------------------------------------
+# Set to true if you have multiple concurrent instances running off the
+# same DB (e.g. Kubernetes pods / load balanced instances). Leave false
+# otherwise.
+
+ha: false
+
 # ---------------------------------------------------------------------
 # Data Path
 # ---------------------------------------------------------------------

dev/build/config.yml

@@ -16,3 +16,4 @@ ssl:
   domain: $(LETSENCRYPT_DOMAIN)
   subscriberEmail: $(LETSENCRYPT_EMAIL)
 logLevel: info
+ha: $(HA_ACTIVE)

dev/index.js

@@ -5,7 +5,6 @@
 // Licensed under AGPLv3
 // ===========================================
 
-const Promise = require('bluebird')
 const _ = require('lodash')
 const chalk = require('chalk')
 
@@ -60,16 +59,9 @@ const init = {
     })
   },
   async reload() {
-    console.warn(chalk.yellow('--- Stopping scheduled jobs...'))
-    if (global.WIKI.scheduler) {
-      global.WIKI.scheduler.stop()
-    }
-    console.warn(chalk.yellow('--- Closing DB connections...'))
-    await global.WIKI.models.knex.destroy()
-    console.warn(chalk.yellow('--- Closing Server connections...'))
-    if (global.WIKI.servers) {
-      await global.WIKI.servers.stopServers()
-    }
+    console.warn(chalk.yellow('--- Gracefully stopping server...'))
+    await global.WIKI.kernel.shutdown()
+
     console.warn(chalk.yellow('--- Purging node modules cache...'))
 
     global.WIKI = {}

package.json

@@ -69,6 +69,7 @@
     "dotize": "0.3.0",
     "elasticsearch6": "npm:@elastic/elasticsearch@6",
     "elasticsearch7": "npm:@elastic/elasticsearch@7",
+    "eventemitter2": "6.0.0",
     "emoji-regex": "9.0.0",
     "express": "4.17.1",
     "express-brute": "1.0.1",
@@ -146,6 +147,7 @@
     "pg": "8.0.2",
     "pg-hstore": "2.3.3",
     "pg-query-stream": "3.0.6",
+    "pg-pubsub": "0.5.0",
     "pg-tsquery": "8.1.0",
     "pug": "2.0.4",
     "punycode": "2.1.1",

server/app/data.yml

@@ -28,6 +28,7 @@ defaults:
       maxFileSize: 5242880
       maxFiles: 10
     offline: false
+    ha: false
     # DB defaults
     api:
       isEnabled: false

server/core/db.js

@@ -17,6 +17,7 @@ const migrateFromBeta = require('../db/beta')
 module.exports = {
   Objection,
   knex: null,
+  listener: null,
   /**
    * Initialize DB
    *
@@ -182,5 +183,55 @@ module.exports = {
       ...this,
       ...models
     }
+  },
+  /**
+   * Subscribe to database LISTEN / NOTIFY for multi-instances events
+   */
+  async subscribeToNotifications () {
+    const useHA = (WIKI.config.ha === true || WIKI.config.ha === 'true' || WIKI.config.ha === 1 || WIKI.config.ha === '1')
+    if (!useHA) {
+      return
+    } else if (WIKI.config.db.type !== 'postgres') {
+      WIKI.logger.warn(`Database engine doesn't support pub/sub. Will not handle concurrent instances: [ DISABLED ]`)
+      return
+    }
+
+    const PGPubSub = require('pg-pubsub')
+
+    this.listener = new PGPubSub(this.knex.client.connectionSettings, {
+      log (ev) {
+        WIKI.logger.debug(ev)
+      }
+    })
+    this.listener.addChannel('wiki', payload => {
+      if (_.has(payload.event) && payload.source !== WIKI.INSTANCE_ID) {
+        WIKI.events.emit(payload.event, payload.value)
+      }
+    })
+    WIKI.events.onAny(this.notifyViaDB)
+
+    WIKI.logger.info(`High-Availability Listener initialized successfully: [ OK ]`)
+  },
+  /**
+   * Unsubscribe from database LISTEN / NOTIFY
+   */
+  async unsubscribeToNotifications () {
+    if (this.listener) {
+      WIKI.events.offAny(this.notifyViaDB)
+      this.listener.close()
+    }
+  },
+  /**
+   * Publish event via database NOTIFY
+   *
+   * @param {string} event Event fired
+   * @param {object} value Payload of the event
+   */
+  notifyViaDB (event, value) {
+    this.listener.publish('wiki', {
+      source: WIKI.INSTANCE_ID,
+      event,
+      value
+    })
   }
 }

server/core/kernel.js

@@ -1,5 +1,5 @@
 const _ = require('lodash')
-const EventEmitter = require('events')
+const EventEmitter = require('eventemitter2').EventEmitter2
 
 /* global WIKI */
 
@@ -37,6 +37,7 @@ module.exports = {
       WIKI.servers = require('./servers')
       WIKI.sideloader = require('./sideloader').init()
       WIKI.events = new EventEmitter()
+      await WIKI.models.subscribeToNotifications()
     } catch (err) {
       WIKI.logger.error(err)
       process.exit(1)
@@ -91,5 +92,21 @@ module.exports = {
       WIKI.logger.warn(err)
       WIKI.telemetry.sendError(err)
     })
+  },
+  /**
+   * Graceful shutdown
+   */
+  async shutdown () {
+    if (WIKI.models) {
+      await WIKI.models.unsubscribeToNotifications()
+      await WIKI.models.knex.client.pool.destroy()
+      await WIKI.models.knex.destroy()
+    }
+    if (WIKI.scheduler) {
+      WIKI.scheduler.stop()
+    }
+    if (WIKI.servers) {
+      await WIKI.servers.stopServers()
+    }
   }
 }

server/index.js

@@ -4,11 +4,13 @@
 // ===========================================
 
 const path = require('path')
+const nanoid = require('nanoid')
 
 let WIKI = {
   IS_DEBUG: process.env.NODE_ENV === 'development',
   IS_MASTER: true,
   ROOTPATH: process.cwd(),
+  INSTANCE_ID: nanoid(10),
   SERVERPATH: path.join(process.cwd(), 'server'),
   Error: require('./helpers/error'),
   configSvc: require('./core/config'),

yarn.lock

@@ -2401,6 +2401,19 @@
   resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-1.19.1.tgz#33509849f8e679e4add158959fdb086440e9553f"
   integrity sha512-5qOlnZscTn4xxM5MeGXAMOsIOIKIbh9e85zJWfBRVPlRMEVawzoPhINYbRGkBZCI8LxvBe7tJCdWiarA99OZfQ==
 
+"@types/pg-types@*":
+  version "1.11.5"
+  resolved "https://registry.yarnpkg.com/@types/pg-types/-/pg-types-1.11.5.tgz#1eebbe62b6772fcc75c18957a90f933d155e005b"
+  integrity sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ==
+
+"@types/pg@^7.4.14":
+  version "7.14.1"
+  resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.14.1.tgz#40358b57c34970f750f6a26e2a5463c9f4758136"
+  integrity sha512-gQgg4bLuykokypx4O1fwEzl5e6UjjyaBtN3znn5zhm0YB9BnKyHDw+e4cQY9rAPzpdM2qpJbn9TNzUazbmTsdw==
+  dependencies:
+    "@types/node" "*"
+    "@types/pg-types" "*"
+
 "@types/q@^1.5.1":
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.2.tgz#690a1475b84f2a884fd07cd797c00f5f31356ea8"
@@ -6862,6 +6875,11 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
+eventemitter2@6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.0.0.tgz#218eb512c3603c5341724b6af7b686a1aa5ab8f5"
+  integrity sha512-ZuNWHD7S7IoikyEmx35vPU8H1W0L+oi644+4mSTg7nwXvBQpIwQL7DPjYUF0VMB0jPkNMo3MqD07E7MYrkFmjQ==
+
 eventemitter3@^3.1.0:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
@@ -11709,6 +11727,11 @@ pg-cursor@^2.1.9:
   resolved "https://registry.yarnpkg.com/pg-cursor/-/pg-cursor-2.1.9.tgz#ad73bd7b119db98d9da6bb148162d831d8cddc88"
   integrity sha512-ReAyfc0fiSzO268DZWtOoeUmMdMG368e7qu3zfwa7TpCjEmabk2XfZn+tdE23XzVlJ4OsdsFcxjASOWC0SS7jA==
 
+pg-format@^1.0.2:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/pg-format/-/pg-format-1.0.4.tgz#27734236c2ad3f4e5064915a59334e20040a828e"
+  integrity sha1-J3NCNsKtP05QZJFaWTNOIAQKgo4=
+
 pg-hstore@2.3.3:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/pg-hstore/-/pg-hstore-2.3.3.tgz#d1978c12a85359830b1388d3b0ff233b88928e96"
@@ -11735,6 +11758,21 @@ pg-query-stream@3.0.6:
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/pg-query-stream/-/pg-query-stream-3.0.6.tgz#12f405c2c8c9723d8d9f1616cf3d58ecd1d87c83"
   integrity sha512-/caOI36GVCz1pY35SkftzGowwym4p39e3Ku+sx8MZKNNf+G9WgE0h+Ui9FHTVV9HWf6WWu1GYt5aYfw5ZMeJsQ==
+pg-pubsub@0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/pg-pubsub/-/pg-pubsub-0.5.0.tgz#5469737af32ac6d13fc3153dc3944f55da3d8840"
+  integrity sha512-lfNMbiVt6d8yomYOWV9+smMdrdUUj3Q8Uq9SiNL+3q2G+jjjjaT9+FKDudw9VEN94pYmCtJzhXz/BlaD4LgyZg==
+  dependencies:
+    "@types/pg" "^7.4.14"
+    pg "^7.4.3"
+    pg-format "^1.0.2"
+    promised-retry "^0.3.0"
+    verror "^1.10.0"
+
+pg-query-stream@3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/pg-query-stream/-/pg-query-stream-3.0.0.tgz#b2b4c3d5eb105df6cf75d3f95c4e157dd527a2ab"
+  integrity sha512-yxeFKwVCW0vmFYSkygV7hd4KVlGCMHGljSUQvYcIZUtfUaAITIl8QOq9oXfCMmk0L6JOuHesZKpurxyuOU8gKg==
   dependencies:
     pg-cursor "^2.1.9"
 
@@ -11768,6 +11806,20 @@ pg@8.0.2:
     pgpass "1.x"
     semver "4.3.2"
 
+pg@^7.4.3:
+  version "7.18.1"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-7.18.1.tgz#67f59c47a99456fcb34f9fe53662b79d4a992f6d"
+  integrity sha512-1KtKBKg/zWrjEEv//klBbVOPGucuc7HHeJf6OEMueVcUeyF3yueHf+DvhVwBjIAe9/97RAydO/lWjkcMwssuEw==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "0.1.3"
+    pg-packet-stream "^1.1.0"
+    pg-pool "^2.0.10"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+    semver "4.3.2"
+
 pgpass@1.x:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306"
@@ -12944,6 +12996,11 @@ promise@^7.0.1:
   dependencies:
     asap "~2.0.3"
 
+promised-retry@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/promised-retry/-/promised-retry-0.3.0.tgz#fc3930ca1f76c4d1e559b7368b1f7e953ae0d6ad"
+  integrity sha512-oj7GS7q3g381QuIR8mMJeJe5GSKm16xXL7Q+7/+5MH2mlWBHXn7dUYmS/BBjGkoBm9M8h2KcVifj3vRZ62AB5A==
+
 prompts@^2.0.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.2.1.tgz#f901dd2a2dfee080359c0e20059b24188d75ad35"
@@ -15788,7 +15845,7 @@ vendors@^1.0.0:
   resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.3.tgz#a6467781abd366217c050f8202e7e50cc9eef8c0"
   integrity sha512-fOi47nsJP5Wqefa43kyWSg80qF+Q3XA6MUkgi7Hp1HQaKDQW4cQrK2D0P7mmbFtsV1N89am55Yru/nyEwRubcw==
 
-verror@1.10.0, verror@^1.8.1:
+verror@1.10.0, verror@^1.10.0, verror@^1.8.1:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400"
   integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=