feat: rendering pipeline logic

client/components/common/nav-header.vue

@@ -92,7 +92,7 @@
         v-btn(icon, slot='activator')
           v-icon(color='grey') account_circle
         span Account
-      v-list.py-0
+      v-list.py-0(:light='!$vuetify.dark')
         v-list-tile.py-3(avatar)
           v-list-tile-avatar
             v-avatar.red(:size='40'): span.white--text.subheading JD

server/graph/resolvers/localization.js

@@ -13,7 +13,7 @@ module.exports = {
   LocalizationQuery: {
     async locales(obj, args, context, info) {
       let remoteLocales = await WIKI.redis.get('locales')
-      let localLocales = await WIKI.models.locales.query().select('id', 'code', 'isRTL', 'name', 'nativeName', 'createdAt', 'updatedAt')
+      let localLocales = await WIKI.models.locales.query().select('code', 'isRTL', 'name', 'nativeName', 'createdAt', 'updatedAt')
       remoteLocales = (remoteLocales) ? JSON.parse(remoteLocales) : localLocales
       return _.map(remoteLocales, rl => {
         let isInstalled = _.some(localLocales, ['code', rl.code])

server/jobs/render-page.js

@@ -1,16 +1,30 @@
 require('../core/worker')
 
+const _ = require('lodash')
+
 /* global WIKI */
 
 WIKI.models = require('../core/db').init()
 
 module.exports = async (job) => {
-  WIKI.logger.info(`Rendering page ${job.data.path}...`)
+  WIKI.logger.info(`Rendering page ${job.data.page.path}...`)
 
   try {
-    WIKI.logger.info(`Rendering page ${job.data.path}: [ COMPLETED ]`)
+    let output = job.data.page.content
+    for (let core of job.data.pipeline) {
+      const renderer = require(`../modules/rendering/${_.kebabCase(core.key)}/renderer.js`)
+      output = await renderer.render.call({
+        config: core.config,
+        children: core.children,
+        page: job.data.page,
+        input: output
+      })
+    }
+    console.info(output)
+
+    WIKI.logger.info(`Rendering page ${job.data.page.path}: [ COMPLETED ]`)
   } catch (err) {
-    WIKI.logger.error(`Rendering page ${job.data.path}: [ FAILED ]`)
+    WIKI.logger.error(`Rendering page ${job.data.page.path}: [ FAILED ]`)
     WIKI.logger.error(err.message)
   }
 }

server/models/pages.js

@@ -111,7 +111,6 @@ module.exports = class Page extends Model {
   }
 
   static async createPage(opts) {
-    await WIKI.models.pages.renderPage(opts)
     const page = await WIKI.models.pages.query().insertAndFetch({
       authorId: opts.authorId,
       content: opts.content,
@@ -127,6 +126,7 @@ module.exports = class Page extends Model {
       publishStartDate: opts.publishStartDate,
       title: opts.title
     })
+    await WIKI.models.pages.renderPage(page)
     await WIKI.models.storage.pageEvent({
       event: 'created',
       page
@@ -149,6 +149,7 @@ module.exports = class Page extends Model {
       publishStartDate: opts.publishStartDate,
       title: opts.title
     })
+    await WIKI.models.pages.renderPage(page)
     await WIKI.models.storage.pageEvent({
       event: 'updated',
       page
@@ -156,8 +157,12 @@ module.exports = class Page extends Model {
     return page
   }
 
-  static async renderPage(opts) {
-    WIKI.queue.job.renderPage.add(opts, {
+  static async renderPage(page) {
+    const pipeline = await WIKI.models.renderers.getRenderingPipeline(page.contentType)
+    WIKI.queue.job.renderPage.add({
+      page,
+      pipeline
+    }, {
       removeOnComplete: true,
       removeOnFail: true
     })

server/models/renderers.js

@@ -3,6 +3,7 @@ const path = require('path')
 const fs = require('fs-extra')
 const _ = require('lodash')
 const yaml = require('js-yaml')
+const DepGraph = require('dependency-graph').DepGraph
 const commonHelper = require('../helpers/common')
 
 /* global WIKI */
@@ -37,10 +38,10 @@ module.exports = class Renderer extends Model {
       const dbRenderers = await WIKI.models.renderers.query()
 
       // -> Fetch definitions from disk
-      const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/renderer'))
+      const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/rendering'))
       let diskRenderers = []
       for (let dir of rendererDirs) {
-        const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/renderer', dir, 'definition.yml'), 'utf8')
+        const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/rendering', dir, 'definition.yml'), 'utf8')
         diskRenderers.push(yaml.safeLoad(def))
       }
       WIKI.data.renderers = diskRenderers.map(renderer => ({
@@ -91,18 +92,66 @@ module.exports = class Renderer extends Model {
     }
   }
 
-  static async pageEvent({ event, page }) {
-    const targets = await WIKI.models.storage.query().where('isEnabled', true)
-    if (targets && targets.length > 0) {
-      _.forEach(targets, target => {
-        WIKI.queue.job.syncStorage.add({
-          event,
-          target,
-          page
-        }, {
-          removeOnComplete: true
+  static async getRenderingPipeline(contentType) {
+    const renderersDb = await WIKI.models.renderers.query().where('isEnabled', true)
+    if (renderersDb && renderersDb.length > 0) {
+      const renderers = renderersDb.map(rdr => {
+        const renderer = _.find(WIKI.data.renderers, ['key', rdr.key])
+        return {
+          ...renderer,
+          config: rdr.config
+        }
+      })
+
+      // Build tree
+      const rawCores = _.filter(renderers, renderer => !_.has(renderer, 'dependsOn')).map(core => {
+        core.children = _.concat([_.cloneDeep(core)], _.filter(renderers, ['dependsOn', core.key]))
+        return core
+      })
+
+      // Build dependency graph
+      const graph = new DepGraph({ circular: true })
+      rawCores.map(core => { graph.addNode(core.key) })
+      rawCores.map(core => {
+        rawCores.map(coreTarget => {
+          if (core.key !== coreTarget.key) {
+            if (core.output === coreTarget.input) {
+              graph.addDependency(core.key, coreTarget.key)
+            }
+          }
         })
       })
+
+      // Filter unused cores
+      let activeCoreKeys = _.filter(rawCores, ['input', contentType]).map(core => core.key)
+      _.clone(activeCoreKeys).map(coreKey => {
+        activeCoreKeys = _.union(activeCoreKeys, graph.dependenciesOf(coreKey))
+      })
+      const activeCores = _.filter(rawCores, core => _.includes(activeCoreKeys, core.key))
+
+      // Rebuild dependency graph with active cores
+      const graphActive = new DepGraph({ circular: true })
+      activeCores.map(core => { graphActive.addNode(core.key) })
+      activeCores.map(core => {
+        activeCores.map(coreTarget => {
+          if (core.key !== coreTarget.key) {
+            if (core.output === coreTarget.input) {
+              graphActive.addDependency(core.key, coreTarget.key)
+            }
+          }
+        })
+      })
+
+      // Reorder cores in reverse dependency order
+      let orderedCores = []
+      _.reverse(graphActive.overallOrder()).map(coreKey => {
+        orderedCores.push(_.find(rawCores, ['key', coreKey]))
+      })
+
+      return orderedCores
+    } else {
+      WIKI.logger.error(`Rendering pipeline is empty!`)
+      return false
     }
   }
 }

server/modules/renderer/html-security/definition.yml

@@ -1,8 +0,0 @@
-key: htmlSecurity
-title: Security
-description: Filter and strips potentially dangerous content
-author: requarks.io
-icon: whatshot
-enabledDefault: true
-dependsOn: htmlCore
-props: {}

server/modules/renderer/markdown-core/definition.yml

@@ -1,31 +0,0 @@
-key: markdownCore
-title: Core
-description: Basic Markdown Parser
-author: requarks.io
-input: markdown
-output: html
-icon: crop_free
-props:
-  linkify:
-    type: Boolean
-    default: true
-    title: Automatically convert links
-    hint: Links will automatically be converted to clickable links.
-  linebreaks:
-    type: Boolean
-    default: true
-    title: Automatically convert line breaks
-    hint: Add linebreaks within paragraphs.
-  highlightCode:
-    type: Boolean
-    default: true
-    title: Highlight code blocks
-    hint: Add syntax coloring to code blocks.
-  codeTheme:
-    type: String
-    default: light
-    title: Code Color Theme
-    hint: Color theme for code blocks
-    enum:
-      - light
-      - dark

server/modules/renderer/markdown-mermaid/renderer.js

@@ -1 +0,0 @@
-module.exports = {}

server/modules/renderer/markdown-plantuml/renderer.js

@@ -1 +0,0 @@
-module.exports = {}

server/modules/rendering/html-core/renderer.js

@@ -0,0 +1,5 @@
+module.exports = {
+  async render() {
+    return this.input
+  }
+}

server/modules/rendering/html-security/definition.yml

@@ -0,0 +1,18 @@
+key: htmlSecurity
+title: Security
+description: Filter and strips potentially dangerous content
+author: requarks.io
+icon: whatshot
+enabledDefault: true
+dependsOn: htmlCore
+props:
+  stripJS:
+    type: Boolean
+    title: Strip Javascript
+    default: false
+    hint: Javascript code within code blocks won't be affected
+  filterBadWords:
+    type: Boolean
+    title: Filter Bad Words
+    default: false
+    hint: Replace bad words with asterisks

server/modules/renderer/markdown-abbr/renderer.js → server/modules/rendering/markdown-abbr/renderer.js

@@ -5,10 +5,6 @@ const mdAbbr = require('markdown-it-abbr')
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/abbreviations',
-  title: 'Abbreviations',
-  dependsOn: [],
-  props: [],
   init (md, conf) {
     md.use(mdAbbr)
   }

server/modules/rendering/markdown-core/definition.yml

@@ -0,0 +1,51 @@
+key: markdownCore
+title: Core
+description: Basic Markdown Parser
+author: requarks.io
+input: markdown
+output: html
+icon: crop_free
+props:
+  allowHTML:
+    type: Boolean
+    default: true
+    title: Allow HTML
+    hint: Enable HTML tags in content
+  linkify:
+    type: Boolean
+    default: true
+    title: Automatically convert links
+    hint: Links will automatically be converted to clickable links.
+  linebreaks:
+    type: Boolean
+    default: true
+    title: Automatically convert line breaks
+    hint: Add linebreaks within paragraphs.
+  typographer:
+    type: Boolean
+    default: false
+    title: Typographer
+    hint: Enable some language-neutral replacement + quotes beautification
+  quotes:
+    type: String
+    default: English
+    title: Quotes style
+    hint: When typographer is enabled. Double + single quotes replacement pairs. e.g. «»„“ for Russian, „“‚‘ for German, etc.
+    enum:
+      - Chinese
+      - English
+      - French
+      - German
+      - Greek
+      - Japanese
+      - Hungarian
+      - Polish
+      - Portuguese
+      - Russian
+      - Spanish
+      - Swedish
+  highlightCode:
+    type: Boolean
+    default: true
+    title: Highlight code blocks
+    hint: Add syntax coloring to code blocks.

server/modules/rendering/markdown-core/renderer.js

@@ -0,0 +1,42 @@
+const md = require('markdown-it')
+// const hljs = require('highlight.js')
+const _ = require('lodash')
+
+const quoteStyles = {
+  Chinese: '””‘’',
+  English: '“”‘’',
+  French: ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'],
+  German: '„“‚‘',
+  Greek: '«»‘’',
+  Japanese: '「」「」',
+  Hungarian: '„”’’',
+  Polish: '„”‚‘',
+  Portuguese: '«»‘’',
+  Russian: '«»„“',
+  Spanish: '«»‘’',
+  Swedish: '””’’'
+}
+
+module.exports = {
+  async render() {
+    const mkdown = md({
+      html: this.config.allowHTML,
+      breaks: this.config.linebreaks,
+      linkify: this.config.linkify,
+      typographer: this.config.typographer,
+      quotes: _.get(quoteStyles, this.config.quotes, quoteStyles.English),
+      highlight(str, lang) {
+        // if (this.config.highlightCode && lang && hljs.getLanguage(lang)) {
+        //   try {
+        //     return '<pre class="hljs"><code>' + hljs.highlight(lang, str, true).value + '</code></pre>'
+        //   } catch (err) {
+        //     return '<pre><code>' + _.escape(str) + '</code></pre>'
+        //   }
+        // }
+        return '<pre><code>' + _.escape(str) + '</code></pre>'
+      }
+    })
+
+    return mkdown.render(this.input)
+  }
+}

server/modules/renderer/markdown-emoji/renderer.js → server/modules/rendering/markdown-emoji/renderer.js

@@ -5,10 +5,6 @@ const mdEmoji = require('markdown-it-emoji')
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/emoji',
-  title: 'Emoji',
-  dependsOn: [],
-  props: [],
   init (md, conf) {
     md.use(mdEmoji)
   }

server/modules/renderer/markdown-expandtabs/definition.yml → server/modules/rendering/markdown-expandtabs/definition.yml

@@ -5,4 +5,9 @@ author: requarks.io
 icon: space_bar
 enabledDefault: true
 dependsOn: markdownCore
-props: {}
+props:
+  tabWidth:
+    type: Number
+    title: Tab Width
+    hint: Amount of spaces for each tab
+    default: 4

server/modules/renderer/markdown-expandtabs/renderer.js → server/modules/rendering/markdown-expandtabs/renderer.js

@@ -6,10 +6,6 @@ const _ = require('lodash')
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/expand-tabs',
-  title: 'Expand Tabs',
-  dependsOn: [],
-  props: ['tabWidth'],
   init (md, conf) {
     md.use(mdExpandTabs, {
       tabWidth: _.toInteger(conf.tabWidth || 4)

server/modules/renderer/markdown-footnotes/renderer.js → server/modules/rendering/markdown-footnotes/renderer.js

@@ -5,10 +5,6 @@ const mdFootnote = require('markdown-it-footnote')
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/footnotes',
-  title: 'Footnotes',
-  dependsOn: [],
-  props: [],
   init (md, conf) {
     md.use(mdFootnote)
   }

server/modules/renderer/markdown-mathjax/renderer.js → server/modules/rendering/markdown-mathjax/renderer.js

@@ -5,10 +5,6 @@ const mdMathjax = require('markdown-it-mathjax')()
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/mathjax',
-  title: 'Mathjax Preprocessor',
-  dependsOn: [],
-  props: [],
   init (md, conf) {
     md.use(mdMathjax)
   }

server/modules/renderer/markdown-tasklists/renderer.js → server/modules/rendering/markdown-tasklists/renderer.js

@@ -5,10 +5,6 @@ const mdTaskLists = require('markdown-it-task-lists')
 // ------------------------------------
 
 module.exports = {
-  key: 'markdown/task-lists',
-  title: 'Task Lists',
-  dependsOn: [],
-  props: [],
   init (md, conf) {
     md.use(mdTaskLists)
   }