feat: cluster implementation

config.sample.yml

@@ -4,23 +4,9 @@
 # Full explanation + examples in the documentation:
 # https://docs.requarks.io/wiki/install
 
-# ---------------------------------------------------------------------
-# Title of this site
-# ---------------------------------------------------------------------
-
-title: Wiki
-
-# ---------------------------------------------------------------------
-# Full public path to the site, without the trailing slash
-# ---------------------------------------------------------------------
-# INCLUDE CLIENT PORT IF NOT 80/443!
-
-host: http://localhost
-
 # ---------------------------------------------------------------------
 # Port the main server should listen to (80 by default)
 # ---------------------------------------------------------------------
-# To use process.env.PORT, comment the line below:
 
 port: 80
 
@@ -33,136 +19,23 @@ paths:
   data: ./data
 
 # ---------------------------------------------------------------------
-# Upload Limits
-# ---------------------------------------------------------------------
-# In megabytes (MB)
-
-uploads:
-  maxImageFileSize: 3
-  maxOtherFileSize: 100
-
-# ---------------------------------------------------------------------
-# Site Language
-# ---------------------------------------------------------------------
-# Possible values: en, es, fr, ko, ru or zh
-
-lang: en
-
-# ---------------------------------------------------------------------
-# Site Authentication
-# ---------------------------------------------------------------------
-
-public: false
-
-auth:
-  defaultReadAccess: false
-  local:
-    enabled: true
-  google:
-    enabled: true
-    clientId: GOOGLE_CLIENT_ID
-    clientSecret: GOOGLE_CLIENT_SECRET
-  microsoft:
-    enabled: true
-    clientId: MS_APP_ID
-    clientSecret: MS_APP_SECRET
-  facebook:
-    enabled: false
-    clientId: FACEBOOK_APP_ID
-    clientSecret: FACEBOOK_APP_SECRET
-  github:
-    enabled: false
-    clientId: GITHUB_CLIENT_ID
-    clientSecret: GITHUB_CLIENT_SECRET
-  slack:
-    enabled: false
-    clientId: SLACK_CLIENT_ID
-    clientSecret: SLACK_CLIENT_SECRET
-  ldap:
-    enabled: false
-    url: ldap://serverhost:389
-    bindDn: cn='root'
-    bindCredentials: BIND_PASSWORD
-    searchBase: o=users,o=example.com
-    searchFilter: (uid={{username}})
-    tlsEnabled: false
-    tlsCertPath: C:\example\root_ca_cert.crt
-  azure:
-    enabled: false
-    clientID: APP_ID
-    clientSecret: APP_SECRET_KEY
-    resource: '00000002-0000-0000-c000-000000000000'
-    tenant: 'YOUR_TENANT.onmicrosoft.com'
-
-# ---------------------------------------------------------------------
-# Secret key to use when encrypting sessions
+# Database
 # ---------------------------------------------------------------------
-# Use a long and unique random string (256-bit keys are perfect!)
 
-sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz
+db:
+  host: localhost
+  port: 5432
+  user: wikijs
+  pass: wikijsrocks
+  db: wiki
 
 # ---------------------------------------------------------------------
-# Database Connection String
+# Redis
 # ---------------------------------------------------------------------
-# You can also use an ENV variable by using $ENV_VAR_NAME as the value
 
-db: mongodb://localhost:27017/wiki
-
-# ---------------------------------------------------------------------
-# Git Connection Info
-# ---------------------------------------------------------------------
-
-git:
-  url: https://github.com/Organization/Repo
-  branch: master
-  auth:
-
-    # Type: basic or ssh
-    type: ssh
-
-    # Only for Basic authentication:
-    username: marty
-    password: MartyMcFly88
-
-    # Only for SSH authentication:
-    privateKey: /etc/wiki/keys/git.pem
-
-    sslVerify: true
-
-  # Default email to use as commit author
-  serverEmail: marty@example.com
-
-  # Whether to use user email as author in commits
-  showUserEmail: true
-
-# ---------------------------------------------------------------------
-# Features
-# ---------------------------------------------------------------------
-# You can enable / disable specific features below
-
-features:
-  linebreaks: true
-  mathjax: true
-
-# ---------------------------------------------------------------------
-# External Logging
-# ---------------------------------------------------------------------
-
-externalLogging:
-  bugsnag: false
-  loggly: false
-  papertrail: false
-  rollbar: false
-  sentry: false
-
-# ---------------------------------------------------------------------
-# Color Theme
-# ---------------------------------------------------------------------
+redis:
+  host: localhost
+  port: 6379
+  db: 0
+  password: null
 
-theme:
-  primary: indigo
-  alt: blue-grey
-  footer: blue-grey
-  code:
-    dark: true
-    colorize: true

package.json

@@ -43,6 +43,7 @@
     "bcryptjs-then": "~1.0.1",
     "bluebird": "~3.5.0",
     "body-parser": "~1.17.2",
+    "bull": "/home/nick3.0.0-rc.4",
     "bunyan": "~1.8.10",
     "cheerio": "~1.0.0-rc.2",
     "child-process-promise": "~2.2.1",

server/index.js

@@ -2,7 +2,6 @@
 
 // ===========================================
 // Wiki.js
-// 1.0.1
 // Licensed under AGPLv3
 // ===========================================
 
@@ -28,251 +27,28 @@ wiki.data = appconf.data
 // Load Winston
 // ----------------------------------------
 
-wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER')
-wiki.logger.info('Wiki.js is initializing...')
+wiki.logger = require('./modules/logger')()
 
 // ----------------------------------------
-// Load global modules
+// Start Cluster
 // ----------------------------------------
 
-wiki.disk = require('./modules/disk').init()
-wiki.db = require('./modules/db').init()
-wiki.entries = require('./modules/entries').init()
-wiki.git = require('./modules/git').init(false)
-wiki.lang = require('i18next')
-wiki.mark = require('./modules/markdown')
-wiki.redis = require('./modules/redis').init()
-wiki.search = require('./modules/search').init()
-wiki.upl = require('./modules/uploads').init()
+const cluster = require('cluster')
+const numCPUs = require('os').cpus().length
 
-// ----------------------------------------
-// Load modules
-// ----------------------------------------
-
-const autoload = require('auto-load')
-const bodyParser = require('body-parser')
-const compression = require('compression')
-const cookieParser = require('cookie-parser')
-const express = require('express')
-const favicon = require('serve-favicon')
-const flash = require('connect-flash')
-const fork = require('child_process').fork
-const http = require('http')
-const i18nBackend = require('i18next-node-fs-backend')
-const passport = require('passport')
-const passportSocketIo = require('passport.socketio')
-const session = require('express-session')
-const SessionRedisStore = require('connect-redis')(session)
-const graceful = require('node-graceful')
-const socketio = require('socket.io')
-const graphqlApollo = require('apollo-server-express')
-const graphqlSchema = require('./modules/graphql')
-
-var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
-var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
-
-// ----------------------------------------
-// Define Express App
-// ----------------------------------------
-
-const app = express()
-wiki.app = app
-app.use(compression())
-
-// ----------------------------------------
-// Security
-// ----------------------------------------
-
-app.use(mw.security)
-
-// ----------------------------------------
-// Public Assets
-// ----------------------------------------
-
-app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
-app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
-  index: false,
-  maxAge: '7d'
-}))
-
-// ----------------------------------------
-// Passport Authentication
-// ----------------------------------------
-
-require('./modules/auth')(passport)
-wiki.rights = require('./modules/rights')
-wiki.rights.init()
-
-let sessionStore = new SessionRedisStore({
-  client: wiki.redis
-})
-
-app.use(cookieParser())
-app.use(session({
-  name: 'wikijs.sid',
-  store: sessionStore,
-  secret: wiki.config.sessionSecret,
-  resave: false,
-  saveUninitialized: false
-}))
-app.use(flash())
-app.use(passport.initialize())
-app.use(passport.session())
-
-// ----------------------------------------
-// SEO
-// ----------------------------------------
-
-app.use(mw.seo)
-
-// ----------------------------------------
-// Localization Engine
-// ----------------------------------------
-
-wiki.lang.use(i18nBackend).init({
-  load: 'languageOnly',
-  ns: ['common', 'admin', 'auth', 'errors', 'git'],
-  defaultNS: 'common',
-  saveMissing: false,
-  preload: [wiki.config.lang],
-  lng: wiki.config.lang,
-  fallbackLng: 'en',
-  backend: {
-    loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
-  }
-})
-
-// ----------------------------------------
-// View Engine Setup
-// ----------------------------------------
-
-app.set('views', path.join(wiki.SERVERPATH, 'views'))
-app.set('view engine', 'pug')
-
-app.use(bodyParser.json({ limit: '1mb' }))
-app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
-
-// ----------------------------------------
-// View accessible data
-// ----------------------------------------
-
-app.locals._ = require('lodash')
-app.locals.t = wiki.lang.t.bind(wiki.lang)
-app.locals.moment = require('moment')
-app.locals.moment.locale(wiki.config.lang)
-app.locals.appconfig = wiki.config
-app.use(mw.flash)
-
-// ----------------------------------------
-// Controllers
-// ----------------------------------------
-
-app.use('/', ctrl.auth)
-
-app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
-app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
-app.use('/uploads', mw.auth, ctrl.uploads)
-app.use('/admin', mw.auth, ctrl.admin)
-app.use('/', mw.auth, ctrl.pages)
-
-// ----------------------------------------
-// Error handling
-// ----------------------------------------
-
-app.use(function (req, res, next) {
-  var err = new Error('Not Found')
-  err.status = 404
-  next(err)
-})
-
-app.use(function (err, req, res, next) {
-  res.status(err.status || 500)
-  res.render('error', {
-    message: err.message,
-    error: wiki.IS_DEBUG ? err : {}
-  })
-})
-
-// ----------------------------------------
-// Start HTTP server
-// ----------------------------------------
-
-wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
-
-app.set('port', wiki.config.port)
-var server = http.createServer(app)
-var io = socketio(server)
-
-server.listen(wiki.config.port)
-server.on('error', (error) => {
-  if (error.syscall !== 'listen') {
-    throw error
-  }
-
-  // handle specific listen errors with friendly messages
-  switch (error.code) {
-    case 'EACCES':
-      wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
-      return process.exit(1)
-    case 'EADDRINUSE':
-      wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
-      return process.exit(1)
-    default:
-      throw error
-  }
-})
+if (cluster.isMaster) {
+  wiki.logger.info('Wiki.js is initializing...')
 
-server.on('listening', () => {
-  wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
-})
-
-// ----------------------------------------
-// WebSocket
-// ----------------------------------------
+  require('./master')
 
-io.use(passportSocketIo.authorize({
-  key: 'wikijs.sid',
-  store: sessionStore,
-  secret: wiki.config.sessionSecret,
-  cookieParser,
-  success: (data, accept) => {
-    accept()
-  },
-  fail: (data, message, error, accept) => {
-    accept()
+  for (let i = 0; i < numCPUs; i++) {
+    cluster.fork()
   }
-}))
-
-io.on('connection', ctrl.ws)
-
-// ----------------------------------------
-// Start child processes
-// ----------------------------------------
 
-let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js'))
-
-bgAgent.on('message', m => {
-  if (!m.action) {
-    return
-  }
-
-  switch (m.action) {
-    case 'searchAdd':
-      wiki.search.add(m.content)
-      break
-  }
-})
-
-// ----------------------------------------
-// Graceful shutdown
-// ----------------------------------------
-
-graceful.on('exit', () => {
-  wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
-  bgAgent.kill()
-  wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
-  return global.git.resync().then(() => {
-    wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
-    process.exit()
+  cluster.on('exit', (worker, code, signal) => {
+    wiki.logger.info(`Worker #${worker.id} died.`)
   })
-})
+} else {
+  wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
+  // require('./worker')
+}

server/master.js

@@ -0,0 +1,232 @@
+'use strict'
+
+/* global wiki */
+
+const path = require('path')
+
+// ----------------------------------------
+// Load global modules
+// ----------------------------------------
+
+wiki.disk = require('./modules/disk').init()
+wiki.db = require('./modules/db').init()
+wiki.entries = require('./modules/entries').init()
+wiki.git = require('./modules/git').init(false)
+wiki.lang = require('i18next')
+wiki.mark = require('./modules/markdown')
+wiki.redis = require('./modules/redis').init()
+wiki.search = require('./modules/search').init()
+wiki.upl = require('./modules/uploads').init()
+
+// ----------------------------------------
+// Load modules
+// ----------------------------------------
+
+const autoload = require('auto-load')
+const bodyParser = require('body-parser')
+const compression = require('compression')
+const cookieParser = require('cookie-parser')
+const express = require('express')
+const favicon = require('serve-favicon')
+const flash = require('connect-flash')
+const http = require('http')
+const i18nBackend = require('i18next-node-fs-backend')
+const passport = require('passport')
+const passportSocketIo = require('passport.socketio')
+const session = require('express-session')
+const SessionRedisStore = require('connect-redis')(session)
+const graceful = require('node-graceful')
+const socketio = require('socket.io')
+const graphqlApollo = require('apollo-server-express')
+const graphqlSchema = require('./modules/graphql')
+
+var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
+var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
+
+// ----------------------------------------
+// Define Express App
+// ----------------------------------------
+
+const app = express()
+wiki.app = app
+app.use(compression())
+
+// ----------------------------------------
+// Security
+// ----------------------------------------
+
+app.use(mw.security)
+
+// ----------------------------------------
+// Public Assets
+// ----------------------------------------
+
+app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
+app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
+  index: false,
+  maxAge: '7d'
+}))
+
+// ----------------------------------------
+// Passport Authentication
+// ----------------------------------------
+
+require('./modules/auth')(passport)
+wiki.rights = require('./modules/rights')
+wiki.rights.init()
+
+let sessionStore = new SessionRedisStore({
+  client: wiki.redis
+})
+
+app.use(cookieParser())
+app.use(session({
+  name: 'wikijs.sid',
+  store: sessionStore,
+  secret: wiki.config.sessionSecret,
+  resave: false,
+  saveUninitialized: false
+}))
+app.use(flash())
+app.use(passport.initialize())
+app.use(passport.session())
+
+// ----------------------------------------
+// SEO
+// ----------------------------------------
+
+app.use(mw.seo)
+
+// ----------------------------------------
+// Localization Engine
+// ----------------------------------------
+
+wiki.lang.use(i18nBackend).init({
+  load: 'languageOnly',
+  ns: ['common', 'admin', 'auth', 'errors', 'git'],
+  defaultNS: 'common',
+  saveMissing: false,
+  preload: [wiki.config.lang],
+  lng: wiki.config.lang,
+  fallbackLng: 'en',
+  backend: {
+    loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
+  }
+})
+
+// ----------------------------------------
+// View Engine Setup
+// ----------------------------------------
+
+app.set('views', path.join(wiki.SERVERPATH, 'views'))
+app.set('view engine', 'pug')
+
+app.use(bodyParser.json({ limit: '1mb' }))
+app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
+
+// ----------------------------------------
+// View accessible data
+// ----------------------------------------
+
+app.locals._ = require('lodash')
+app.locals.t = wiki.lang.t.bind(wiki.lang)
+app.locals.moment = require('moment')
+app.locals.moment.locale(wiki.config.lang)
+app.locals.appconfig = wiki.config
+app.use(mw.flash)
+
+// ----------------------------------------
+// Controllers
+// ----------------------------------------
+
+app.use('/', ctrl.auth)
+
+app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
+app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
+app.use('/uploads', mw.auth, ctrl.uploads)
+app.use('/admin', mw.auth, ctrl.admin)
+app.use('/', mw.auth, ctrl.pages)
+
+// ----------------------------------------
+// Error handling
+// ----------------------------------------
+
+app.use(function (req, res, next) {
+  var err = new Error('Not Found')
+  err.status = 404
+  next(err)
+})
+
+app.use(function (err, req, res, next) {
+  res.status(err.status || 500)
+  res.render('error', {
+    message: err.message,
+    error: wiki.IS_DEBUG ? err : {}
+  })
+})
+
+// ----------------------------------------
+// Start HTTP server
+// ----------------------------------------
+
+wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
+
+app.set('port', wiki.config.port)
+var server = http.createServer(app)
+var io = socketio(server)
+
+server.listen(wiki.config.port)
+server.on('error', (error) => {
+  if (error.syscall !== 'listen') {
+    throw error
+  }
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
+      return process.exit(1)
+    case 'EADDRINUSE':
+      wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
+      return process.exit(1)
+    default:
+      throw error
+  }
+})
+
+server.on('listening', () => {
+  wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
+})
+
+// ----------------------------------------
+// WebSocket
+// ----------------------------------------
+
+io.use(passportSocketIo.authorize({
+  key: 'wikijs.sid',
+  store: sessionStore,
+  secret: wiki.config.sessionSecret,
+  cookieParser,
+  success: (data, accept) => {
+    accept()
+  },
+  fail: (data, message, error, accept) => {
+    accept()
+  }
+}))
+
+io.on('connection', ctrl.ws)
+
+// ----------------------------------------
+// Graceful shutdown
+// ----------------------------------------
+
+graceful.on('exit', () => {
+  // wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
+  // bgAgent.kill()
+  wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
+  return global.git.resync().then(() => {
+    wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
+    process.exit()
+  })
+})

server/models/_relations.js

@@ -4,7 +4,10 @@
  * Associate DB Model relations
  */
 module.exports = db => {
-  db.User.belongsToMany(db.Group, { through: 'UserGroups' })
-  db.Group.hasMany(db.Right, { as: 'GroupRights' })
+  db.User.belongsToMany(db.Group, { through: 'userGroups' })
+  db.Group.hasMany(db.Right, { as: 'groupRights' })
+  db.Document.hasMany(db.Tag, { as: 'documentTags' })
   db.File.belongsTo(db.Folder)
+  db.Comment.belongsTo(db.Document)
+  db.Comment.belongsTo(db.User, { as: 'author' })
 }

server/models/comment.js

@@ -0,0 +1,18 @@
+'use strict'
+
+/**
+ * Comment schema
+ */
+module.exports = (sequelize, DataTypes) => {
+  let commentSchema = sequelize.define('comment', {
+    content: {
+      type: DataTypes.STRING,
+      allowNull: false
+    }
+  }, {
+    timestamps: true,
+    version: true
+  })
+
+  return commentSchema
+}

server/models/document.js

@@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.BOOLEAN,
       allowNull: false,
       defaultValue: false
+    },
+    searchContent: {
+      type: DataTypes.TEXT,
+      allowNull: true,
+      defaultValue: ''
     }
   }, {
     timestamps: true,

server/models/tag.js

@@ -0,0 +1,24 @@
+'use strict'
+
+/**
+ * Tags schema
+ */
+module.exports = (sequelize, DataTypes) => {
+  let tagSchema = sequelize.define('tag', {
+    key: {
+      type: DataTypes.STRING,
+      allowNull: false
+    }
+  }, {
+    timestamps: true,
+    version: true,
+    indexes: [
+      {
+        unique: true,
+        fields: ['key']
+      }
+    ]
+  })
+
+  return tagSchema
+}

server/modules/db.js

@@ -64,7 +64,8 @@ module.exports = {
     // Sync DB
 
     self.onReady = self.inst.sync({
-      force: false
+      force: false,
+      logging: false
     })
 
     return self

server/modules/logger.js

@@ -2,12 +2,10 @@
 
 /* global wiki */
 
-module.exports = (processName) => {
-  let winston = require('winston')
+const cluster = require('cluster')
 
-  if (typeof processName === 'undefined') {
-    processName = 'SERVER'
-  }
+module.exports = () => {
+  let winston = require('winston')
 
   // Console
 
@@ -25,6 +23,7 @@ module.exports = (processName) => {
   })
 
   logger.filters.push((level, msg) => {
+    let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
     return '[' + processName + '] ' + msg
   })
 

server/modules/search-index/index.js

@@ -1,81 +0,0 @@
-const bunyan = require('bunyan')
-const level = require('levelup')
-const down = require('memdown')
-const SearchIndexAdder = require('search-index-adder')
-const SearchIndexSearcher = require('search-index-searcher')
-
-module.exports = function (givenOptions, moduleReady) {
-  const optionsLoaded = function (err, SearchIndex) {
-    const siUtil = require('./siUtil.js')(SearchIndex.options)
-    if (err) return moduleReady(err)
-    SearchIndex.close = siUtil.close
-    SearchIndex.countDocs = siUtil.countDocs
-    getAdder(SearchIndex, adderLoaded)
-  }
-
-  const adderLoaded = function (err, SearchIndex) {
-    if (err) return moduleReady(err)
-    getSearcher(SearchIndex, searcherLoaded)
-  }
-
-  const searcherLoaded = function (err, SearchIndex) {
-    if (err) return moduleReady(err)
-    return moduleReady(err, SearchIndex)
-  }
-
-  getOptions(givenOptions, optionsLoaded)
-}
-
-const getAdder = function (SearchIndex, done) {
-  SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
-    SearchIndex.add = searchIndexAdder.add
-    SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
-    SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
-    SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
-    SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
-    SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
-    SearchIndex.del = searchIndexAdder.deleter
-    SearchIndex.deleteStream = searchIndexAdder.deleteStream
-    SearchIndex.flush = searchIndexAdder.flush
-    done(err, SearchIndex)
-  })
-}
-
-const getSearcher = function (SearchIndex, done) {
-  SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
-    SearchIndex.availableFields = searchIndexSearcher.availableFields
-    SearchIndex.buckets = searchIndexSearcher.bucketStream
-    SearchIndex.categorize = searchIndexSearcher.categoryStream
-    SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
-    SearchIndex.get = searchIndexSearcher.get
-    SearchIndex.match = searchIndexSearcher.match
-    SearchIndex.scan = searchIndexSearcher.scan
-    SearchIndex.search = searchIndexSearcher.search
-    SearchIndex.totalHits = searchIndexSearcher.totalHits
-    done(err, SearchIndex)
-  })
-}
-
-const getOptions = function (options, done) {
-  var SearchIndex = {}
-  SearchIndex.options = Object.assign({}, {
-    indexPath: 'si',
-    keySeparator: '￮',
-    logLevel: 'error'
-  }, options)
-  options.log = bunyan.createLogger({
-    name: 'search-index',
-    level: options.logLevel
-  })
-  if (!options.indexes) {
-    level(SearchIndex.options.indexPath || 'si', {
-      valueEncoding: 'json',
-      db: down
-    }, function (err, db) {
-      SearchIndex.options.indexes = db
-      return done(err, SearchIndex)
-    })
-  } else {
-    return done(null, SearchIndex)
-  }
-}

server/modules/search-index/siUtil.js

@@ -1,36 +0,0 @@
-'use strict'
-
-module.exports = function (siOptions) {
-  var siUtil = {}
-
-  siUtil.countDocs = function (callback) {
-    var count = 0
-    const gte = 'DOCUMENT' + siOptions.keySeparator
-    const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
-    siOptions.indexes.createReadStream({gte: gte, lte: lte})
-      .on('data', function (data) {
-        count++
-      })
-      .on('error', function (err) {
-        return callback(err, null)
-      })
-      .on('end', function () {
-        return callback(null, count)
-      })
-  }
-
-  siUtil.close = function (callback) {
-    siOptions.indexes.close(function (err) {
-      while (!siOptions.indexes.isClosed()) {
-        // log not always working here- investigate
-        if (siOptions.log) siOptions.log.info('closing...')
-      }
-      if (siOptions.indexes.isClosed()) {
-        if (siOptions.log) siOptions.log.info('closed...')
-        callback(err)
-      }
-    })
-  }
-
-  return siUtil
-}

server/modules/search.js

@@ -4,7 +4,7 @@
 
 const Promise = require('bluebird')
 const _ = require('lodash')
-const searchIndex = require('./search-index')
+// const searchIndex = require('./search-index')
 const stopWord = require('stopword')
 const streamToPromise = require('stream-to-promise')
 const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
@@ -22,7 +22,7 @@ module.exports = {
   init () {
     let self = this
     self._isReady = new Promise((resolve, reject) => {
-      searchIndex({
+      /*searchIndex({
         deletable: true,
         fieldedSearch: true,
         indexPath: 'wiki',
@@ -39,7 +39,7 @@ module.exports = {
             resolve(true)
           })
         }
-      })
+      }) */
     })
 
     return self

server/agent.js → server/worker.js

@@ -1,39 +1,19 @@
-// ===========================================
-// Wiki.js - Background Agent
-// 1.0.1
-// Licensed under AGPLv3
-// ===========================================
+'use strict'
 
-const path = require('path')
-const ROOTPATH = process.cwd()
-const SERVERPATH = path.join(ROOTPATH, 'server')
-
-global.ROOTPATH = ROOTPATH
-global.SERVERPATH = SERVERPATH
-const IS_DEBUG = process.env.NODE_ENV === 'development'
-
-let appconf = require('./modules/config')()
-global.appconfig = appconf.config
-global.appdata = appconf.data
-
-// ----------------------------------------
-// Load Winston
-// ----------------------------------------
+/* global wiki */
 
-global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT')
+const path = require('path')
 
 // ----------------------------------------
 // Load global modules
 // ----------------------------------------
 
-global.winston.info('Background Agent is initializing...')
-
-global.db = require('./modules/db').init()
-global.upl = require('./modules/uploads-agent').init()
-global.git = require('./modules/git').init()
-global.entries = require('./modules/entries').init()
-global.lang = require('i18next')
-global.mark = require('./modules/markdown')
+wiki.db = require('./modules/db').init()
+wiki.upl = require('./modules/uploads-agent').init()
+wiki.git = require('./modules/git').init()
+wiki.entries = require('./modules/entries').init()
+wiki.lang = require('i18next')
+wiki.mark = require('./modules/markdown')
 
 // ----------------------------------------
 // Load modules
@@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry')
 // Localization Engine
 // ----------------------------------------
 
-global.lang
-  .use(i18nBackend)
-  .init({
-    load: 'languageOnly',
-    ns: ['common', 'admin', 'auth', 'errors', 'git'],
-    defaultNS: 'common',
-    saveMissing: false,
-    preload: [appconfig.lang],
-    lng: appconfig.lang,
-    fallbackLng: 'en',
-    backend: {
-      loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
-    }
-  })
+wiki.lang.use(i18nBackend).init({
+  load: 'languageOnly',
+  ns: ['common', 'admin', 'auth', 'errors', 'git'],
+  defaultNS: 'common',
+  saveMissing: false,
+  preload: [wiki.config.lang],
+  lng: wiki.config.lang,
+  fallbackLng: 'en',
+  backend: {
+    loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
+  }
+})
 
 // ----------------------------------------
 // Start Cron
@@ -75,8 +53,8 @@ let job
 let jobIsBusy = false
 let jobUplWatchStarted = false
 
-global.db.onReady.then(() => {
-  return global.db.Entry.remove({})
+wiki.db.onReady.then(() => {
+  return wiki.db.Entry.remove({})
 }).then(() => {
   job = new Cron({
     cronTime: '0 */5 * * * *',
@@ -84,17 +62,17 @@ global.db.onReady.then(() => {
       // Make sure we don't start two concurrent jobs
 
       if (jobIsBusy) {
-        global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
+        wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
         return
       }
-      global.winston.info('Running all jobs...')
+      wiki.logger.info('Running all jobs...')
       jobIsBusy = true
 
       // Prepare async job collector
 
       let jobs = []
-      let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo)
-      let dataPath = path.resolve(ROOTPATH, appconfig.paths.data)
+      let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
+      let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
       let uploadsTempPath = path.join(dataPath, 'temp-upload')
 
       // ----------------------------------------
@@ -105,7 +83,7 @@ global.db.onReady.then(() => {
       // -> Sync with Git remote
       //* ****************************************
 
-      jobs.push(global.git.resync().then(() => {
+      jobs.push(wiki.git.resync().then(() => {
         // -> Stream all documents
 
         let cacheJobs = []
@@ -185,18 +163,18 @@ global.db.onReady.then(() => {
       // ----------------------------------------
 
       Promise.all(jobs).then(() => {
-        global.winston.info('All jobs completed successfully! Going to sleep for now.')
+        wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
 
         if (!jobUplWatchStarted) {
           jobUplWatchStarted = true
-          global.upl.initialScan().then(() => {
+          wiki.upl.initialScan().then(() => {
             job.start()
           })
         }
 
         return true
       }).catch((err) => {
-        global.winston.error('One or more jobs have failed: ', err)
+        wiki.logger.error('One or more jobs have failed: ', err)
       }).finally(() => {
         jobIsBusy = false
       })
@@ -212,7 +190,7 @@ global.db.onReady.then(() => {
 // ----------------------------------------
 
 process.on('disconnect', () => {
-  global.winston.warn('Lost connection to main server. Exiting...')
+  wiki.logger.warn('Lost connection to main server. Exiting...')
   job.stop()
   process.exit()
 })

yarn.lock

@@ -1036,7 +1036,7 @@ block-stream@*:
   dependencies:
     inherits "~2.0.0"
 
-bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@~3.5.0:
+bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@~3.5.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c"
 
@@ -1124,6 +1124,18 @@ builtin-modules@^1.0.0, builtin-modules@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
 
+bull@/home/nick3.0.0-rc.4:
+  version "3.0.0-rc.4"
+  resolved "https://registry.yarnpkg.com/bull/-/bull-3.0.0-rc.4.tgz#dea18e870787037183849fc0198982ed756589b7"
+  dependencies:
+    bluebird "^3.5.0"
+    cron-parser "^2.4.1"
+    debuglog "^1.0.0"
+    ioredis "^3.1.1"
+    lodash "^4.17.4"
+    semver "^5.3.0"
+    uuid "^3.1.0"
+
 bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10:
   version "1.8.10"
   resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c"
@@ -1590,6 +1602,13 @@ crc@3.4.4, crc@^3.4.0:
   version "3.4.4"
   resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b"
 
+cron-parser@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.4.1.tgz#022befce1af293e4d3144ff04c2cbd2edb491271"
+  dependencies:
+    is-nan "^1.2.1"
+    moment-timezone "^0.5.0"
+
 cron@1.2.1, cron@~1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2"
@@ -1702,6 +1721,10 @@ debug@~2.2.0:
   dependencies:
     ms "0.7.1"
 
+debuglog@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
+
 decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -1732,6 +1755,13 @@ deferred-leveldown@~1.2.1:
   dependencies:
     abstract-leveldown "~2.4.0"
 
+define-properties@^1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94"
+  dependencies:
+    foreach "^2.0.5"
+    object-keys "^1.0.8"
+
 del@^2.0.2:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8"
@@ -2573,6 +2603,10 @@ for-own@^0.1.4:
   dependencies:
     for-in "^1.0.1"
 
+foreach@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -3207,6 +3241,19 @@ invert-kv@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
 
+ioredis@^3.1.1:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.2.tgz#2579e3eba6dc490f68f14c7b51346281332b467b"
+  dependencies:
+    bluebird "^3.3.4"
+    cluster-key-slot "^1.0.6"
+    debug "^2.2.0"
+    denque "^1.1.0"
+    flexbuffer "0.0.6"
+    lodash "^4.8.2"
+    redis-commands "^1.2.0"
+    redis-parser "^2.4.0"
+
 ioredis@~3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8"
@@ -3320,6 +3367,12 @@ is-glob@^2.0.0, is-glob@^2.0.1:
   dependencies:
     is-extglob "^1.0.0"
 
+is-nan@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.2.1.tgz#9faf65b6fb6db24b7f5c0628475ea71f988401e2"
+  dependencies:
+    define-properties "^1.1.1"
+
 is-npm@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"
@@ -4574,7 +4627,7 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
   dependencies:
     minimist "0.0.8"
 
-moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
+moment-timezone@^0.5.0, moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
   version "0.5.13"
   resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90"
   dependencies:
@@ -4877,6 +4930,10 @@ object-component@0.0.3:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"
 
+object-keys@^1.0.8:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
+
 object.omit@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@@ -7045,7 +7102,7 @@ uuid@^2.0.1, uuid@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
 
-uuid@^3.0.0, uuid@^3.0.1:
+uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"