feat: scheduler - regular cron check + add future jobs

dev/templates/base.pug

@@ -38,20 +38,6 @@ html(lang=siteConfig.lang)
       script.
         siteConfig.devMode = true
 
-    //- Icon Set
-    if config.theming.iconset === 'fa'
-      link(
-        type='text/css'
-        rel='stylesheet'
-        href='https://use.fontawesome.com/releases/v5.10.0/css/all.css'
-        )
-    else if config.theming.iconset === 'fa4'
-      link(
-        type='text/css'
-        rel='stylesheet'
-        href='https://cdn.jsdelivr.net/npm/font-awesome@4.7.0/css/font-awesome.min.css'
-        )
-
     //- CSS
     <% for (var index in htmlWebpackPlugin.files.css) { %>
       <% if (htmlWebpackPlugin.files.cssIntegrity) { %>

server/app/data.yml

@@ -31,6 +31,8 @@ defaults:
     bodyParserLimit: 5mb
     scheduler:
       workers: 3
+      pollingCheck: 5
+      scheduledCheck: 300
     # DB defaults
     api:
       isEnabled: false

server/controllers/common.js

@@ -158,15 +158,15 @@ router.get(['/_edit', '/_edit/*'], async (req, res, next) => {
     return res.redirect(`/_edit/home`)
   }
 
-  if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
-    return res.redirect(`/_edit/${pageArgs.locale}/${pageArgs.path}`)
-  }
+  // if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
+  //   return res.redirect(`/_edit/${pageArgs.locale}/${pageArgs.path}`)
+  // }
 
-  req.i18n.changeLanguage(pageArgs.locale)
+  // req.i18n.changeLanguage(pageArgs.locale)
 
   // -> Set Editor Lang
   _.set(res, 'locals.siteConfig.lang', pageArgs.locale)
-  _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
+  // _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
 
   // -> Check for reserved path
   if (pageHelper.isReservedPath(pageArgs.path)) {
@@ -187,9 +187,9 @@ router.get(['/_edit', '/_edit/*'], async (req, res, next) => {
   const effectivePermissions = WIKI.auth.getEffectivePermissions(req, pageArgs)
 
   const injectCode = {
-    css: WIKI.config.theming.injectCSS,
-    head: WIKI.config.theming.injectHead,
-    body: WIKI.config.theming.injectBody
+    css: '', // WIKI.config.theming.injectCSS,
+    head: '', // WIKI.config.theming.injectHead,
+    body: '' // WIKI.config.theming.injectBody
   }
 
   if (page) {
@@ -462,11 +462,11 @@ router.get('/*', async (req, res, next) => {
   const isPage = (stripExt || pageArgs.path.indexOf('.') === -1)
 
   if (isPage) {
-    if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
-      return res.redirect(`/${pageArgs.locale}/${pageArgs.path}`)
-    }
+    // if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
+    //   return res.redirect(`/${pageArgs.locale}/${pageArgs.path}`)
+    // }
 
-    req.i18n.changeLanguage(pageArgs.locale)
+    // req.i18n.changeLanguage(pageArgs.locale)
 
     try {
       // -> Get Page from cache
@@ -494,7 +494,7 @@ router.get('/*', async (req, res, next) => {
       }
 
       _.set(res, 'locals.siteConfig.lang', pageArgs.locale)
-      _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
+      // _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
 
       if (page) {
         _.set(res.locals, 'pageMeta.title', page.title)

server/core/auth.js

@@ -453,9 +453,9 @@ module.exports = {
   getEffectivePermissions (req, page) {
     return {
       comments: {
-        read: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['read:comments'], page) : false,
-        write: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['write:comments'], page) : false,
-        manage: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['manage:comments'], page) : false
+        read: WIKI.auth.checkAccess(req.user, ['read:comments'], page),
+        write: WIKI.auth.checkAccess(req.user, ['write:comments'], page),
+        manage: WIKI.auth.checkAccess(req.user, ['manage:comments'], page)
       },
       history: {
         read: WIKI.auth.checkAccess(req.user, ['read:history'], page)

server/core/scheduler.js

@@ -1,18 +1,21 @@
 const { DynamicThreadPool } = require('poolifier')
 const os = require('node:os')
-const { setTimeout } = require('node:timers/promises')
 const autoload = require('auto-load')
 const path = require('node:path')
+const cronparser = require('cron-parser')
+const { DateTime } = require('luxon')
 
 module.exports = {
-  pool: null,
+  workerPool: null,
   maxWorkers: 1,
   activeWorkers: 0,
+  pollingRef: null,
+  scheduledRef: null,
   tasks: null,
   async init () {
     this.maxWorkers = WIKI.config.scheduler.workers === 'auto' ? os.cpus().length : WIKI.config.scheduler.workers
     WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`)
-    this.pool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
+    this.workerPool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
       errorHandler: (err) => WIKI.logger.warn(err),
       exitHandler: () => WIKI.logger.debug('A worker has gone offline.'),
       onlineHandler: () => WIKI.logger.debug('New worker is online.')
@@ -22,39 +25,68 @@ module.exports = {
   },
   async start () {
     WIKI.logger.info('Starting Scheduler...')
-    WIKI.db.listener.addChannel('scheduler', payload => {
+
+    // -> Add PostgreSQL Sub Channel
+    WIKI.db.listener.addChannel('scheduler', async payload => {
       switch (payload.event) {
         case 'newJob': {
           if (this.activeWorkers < this.maxWorkers) {
             this.activeWorkers++
-            this.processJob()
+            await this.processJob()
+            this.activeWorkers--
           }
           break
         }
       }
     })
-    // await WIKI.db.knex('jobs').insert({
-    //   task: 'test',
-    //   payload: { foo: 'bar' }
-    // })
-    // WIKI.db.listener.publish('scheduler', {
-    //   source: WIKI.INSTANCE_ID,
-    //   event: 'newJob'
-    // })
+
+    // -> Start scheduled jobs check
+    this.scheduledRef = setInterval(async () => {
+      this.addScheduled()
+    }, WIKI.config.scheduler.scheduledCheck * 1000)
+
+    // -> Add scheduled jobs on init
+    await this.addScheduled()
+
+    // -> Start job polling
+    this.pollingRef = setInterval(async () => {
+      this.processJob()
+    }, WIKI.config.scheduler.pollingCheck * 1000)
+
     WIKI.logger.info('Scheduler: [ STARTED ]')
   },
+  async addJob ({ task, payload, waitUntil, isScheduled = false, notify = true }) {
+    try {
+      await WIKI.db.knex('jobs').insert({
+        task,
+        useWorker: !(typeof this.tasks[task] === 'function'),
+        payload,
+        isScheduled,
+        waitUntil,
+        createdBy: WIKI.INSTANCE_ID
+      })
+      if (notify) {
+        WIKI.db.listener.publish('scheduler', {
+          source: WIKI.INSTANCE_ID,
+          event: 'newJob'
+        })
+      }
+    } catch (err) {
+      WIKI.logger.warn(`Failed to add job to scheduler: ${err.message}`)
+    }
+  },
   async processJob () {
     try {
       await WIKI.db.knex.transaction(async trx => {
         const jobs = await trx('jobs')
-          .where('id', WIKI.db.knex.raw('(SELECT id FROM jobs ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1)'))
+          .where('id', WIKI.db.knex.raw('(SELECT id FROM jobs WHERE ("waitUntil" IS NULL OR "waitUntil" <= NOW()) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1)'))
           .returning('*')
           .del()
         if (jobs && jobs.length === 1) {
           const job = jobs[0]
           WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`)
           if (job.useWorker) {
-            await this.pool.execute({
+            await this.workerPool.execute({
               id: job.id,
               name: job.task,
               data: job.payload
@@ -68,9 +100,74 @@ module.exports = {
       WIKI.logger.warn(err)
     }
   },
+  async addScheduled () {
+    try {
+      await WIKI.db.knex.transaction(async trx => {
+        // -> Acquire lock
+        const jobLock = await trx('jobLock')
+          .where(
+            'key',
+            WIKI.db.knex('jobLock')
+              .select('key')
+              .where('key', 'cron')
+              .andWhere('lastCheckedAt', '<=', DateTime.utc().minus({ minutes: 5 }).toISO())
+              .forUpdate()
+              .skipLocked()
+              .limit(1)
+          ).update({
+            lastCheckedBy: WIKI.INSTANCE_ID,
+            lastCheckedAt: DateTime.utc().toISO()
+          })
+        if (jobLock > 0) {
+          WIKI.logger.info(`Scheduling future planned jobs...`)
+          const scheduledJobs = await WIKI.db.knex('jobSchedule')
+          if (scheduledJobs?.length > 0) {
+            // -> Get existing scheduled jobs
+            const existingJobs = await WIKI.db.knex('jobs').where('isScheduled', true)
+            for (const job of scheduledJobs) {
+              // -> Get next planned iterations
+              const plannedIterations = cronparser.parseExpression(job.cron, {
+                startDate: DateTime.utc().toJSDate(),
+                endDate: DateTime.utc().plus({ days: 1, minutes: 5 }).toJSDate(),
+                iterator: true,
+                tz: 'UTC'
+              })
+              // -> Add a maximum of 10 future iterations for a single task
+              let addedFutureJobs = 0
+              while (true) {
+                try {
+                  const next = plannedIterations.next()
+                  // -> Ensure this iteration isn't already scheduled
+                  if (!existingJobs.some(j => j.task === job.task && j.waitUntil.getTime() === next.value.getTime())) {
+                    this.addJob({
+                      task: job.task,
+                      useWorker: !(typeof this.tasks[job.task] === 'function'),
+                      payload: job.payload,
+                      isScheduled: true,
+                      waitUntil: next.value.toISOString(),
+                      notify: false
+                    })
+                    addedFutureJobs++
+                  }
+                  // -> No more iterations for this period or max iterations count reached
+                  if (next.done || addedFutureJobs >= 10) { break }
+                } catch (err) {
+                  break
+                }
+              }
+            }
+          }
+        }
+      })
+    } catch (err) {
+      WIKI.logger.warn(err)
+    }
+  },
   async stop () {
     WIKI.logger.info('Stopping Scheduler...')
-    await this.pool.destroy()
+    clearInterval(this.scheduledRef)
+    clearInterval(this.pollingRef)
+    await this.workerPool.destroy()
     WIKI.logger.info('Scheduler: [ STOPPED ]')
   }
 }

server/db/migrations/3.0.0.js

@@ -1,6 +1,7 @@
 const { v4: uuid } = require('uuid')
 const bcrypt = require('bcryptjs-then')
 const crypto = require('crypto')
+const { DateTime } = require('luxon')
 const pem2jwk = require('pem-jwk').pem2jwk
 
 exports.up = async knex => {
@@ -120,16 +121,6 @@ exports.up = async knex => {
       table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
       table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
     })
-    // JOB SCHEDULE ------------------------
-    .createTable('jobSchedule', table => {
-      table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
-      table.string('task').notNullable()
-      table.string('cron').notNullable()
-      table.string('type').notNullable().defaultTo('system')
-      table.jsonb('payload')
-      table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
-      table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
-    })
     // JOB HISTORY -------------------------
     .createTable('jobHistory', table => {
       table.uuid('id').notNullable().primary()
@@ -141,6 +132,22 @@ exports.up = async knex => {
       table.timestamp('startedAt').notNullable()
       table.timestamp('completedAt').notNullable().defaultTo(knex.fn.now())
     })
+    // JOB SCHEDULE ------------------------
+    .createTable('jobSchedule', table => {
+      table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
+      table.string('task').notNullable()
+      table.string('cron').notNullable()
+      table.string('type').notNullable().defaultTo('system')
+      table.jsonb('payload')
+      table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
+      table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
+    })
+    // JOB SCHEDULE ------------------------
+    .createTable('jobLock', table => {
+      table.string('key').notNullable().primary()
+      table.string('lastCheckedBy')
+      table.timestamp('lastCheckedAt').notNullable().defaultTo(knex.fn.now())
+    })
     // JOBS --------------------------------
     .createTable('jobs', table => {
       table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
@@ -148,6 +155,8 @@ exports.up = async knex => {
       table.boolean('useWorker').notNullable().defaultTo(false)
       table.jsonb('payload')
       table.timestamp('waitUntil')
+      table.boolean('isScheduled').notNullable().defaultTo(false)
+      table.string('createdBy')
       table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
       table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
     })
@@ -680,6 +689,12 @@ exports.up = async knex => {
     }
   ])
 
+  await knex('jobLock').insert({
+    key: 'cron',
+    lastCheckedBy: 'init',
+    lastCheckedAt: DateTime.utc().minus({ hours: 1 }).toISO()
+  })
+
   WIKI.logger.info('Completed 3.0.0 database migration.')
 }
 

server/helpers/page.js

@@ -20,7 +20,8 @@ module.exports = {
    */
   parsePath (rawPath, opts = {}) {
     let pathObj = {
-      locale: WIKI.config.lang.code,
+      // TODO: use site base lang
+      locale: 'en', // WIKI.config.lang.code,
       path: 'home',
       private: false,
       privateNS: '',

server/views/base.pug

@@ -38,20 +38,6 @@ html(lang=siteConfig.lang)
       script.
         siteConfig.devMode = true
 
-    //- Icon Set
-    if config.theming.iconset === 'fa'
-      link(
-        type='text/css'
-        rel='stylesheet'
-        href='https://use.fontawesome.com/releases/v5.10.0/css/all.css'
-        )
-    else if config.theming.iconset === 'fa4'
-      link(
-        type='text/css'
-        rel='stylesheet'
-        href='https://cdn.jsdelivr.net/npm/font-awesome@4.7.0/css/font-awesome.min.css'
-        )
-
     //- CSS
     
       
@@ -68,14 +54,14 @@ html(lang=siteConfig.lang)
       
     script(
       type='text/javascript'
-      src='/_assets-legacy/js/runtime.js?1662846772'
+      src='/_assets-legacy/js/runtime.js?1664769154'
       )
       
     
       
     script(
       type='text/javascript'
-      src='/_assets-legacy/js/app.js?1662846772'
+      src='/_assets-legacy/js/app.js?1664769154'
       )
       
     

server/web.js

@@ -134,7 +134,6 @@ module.exports = async () => {
   // View accessible data
   // ----------------------------------------
 
-  app.locals.siteConfig = {}
   app.locals.analyticsCode = {}
   app.locals.basedir = WIKI.ROOTPATH
   app.locals.config = WIKI.config
@@ -173,6 +172,9 @@ module.exports = async () => {
       rtl: false, // TODO: handle RTL
       company: currentSite.config.company,
       contentLicense: currentSite.config.contentLicense
+    }
+    res.locals.theming = {
+
     }
     res.locals.langs = await WIKI.db.locales.getNavLocales({ cache: true })
     res.locals.analyticsCode = await WIKI.db.analytics.getCode({ cache: true })