Search-Index integration + cache flush on start

.eslintrc.json

@@ -32,6 +32,7 @@
     "lcdata": true,
     "mark": true,
     "rights": true,
+    "search": true,
     "upl": true,
     "winston": true,
     "ws": true,

CHANGELOG.md

@@ -5,6 +5,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## [Unreleased]
 ### Added
 - Offline mode (no remote git sync) can now be enabled by setting `git: false` in config.yml
+- Improved search engine (Now using search-index engine instead of MongoDB text search)
+
+### Changed
+- Cache is now flushed when starting / restarting the server
 
 ## [v1.0-beta.4] - 2017-02-11
 ### Fixed

agent.js

@@ -113,7 +113,13 @@ var job = new Cron({
                 // -> Update cache and search index
 
                 if (fileStatus !== 'active') {
-                  return entries.updateCache(entryPath)
+                  return entries.updateCache(entryPath).then(entry => {
+                    process.send({
+                      action: 'searchAdd',
+                      content: entry
+                    })
+                    return true
+                  })
                 }
 
                 return true

client/js/components/search.js

@@ -42,7 +42,7 @@ if ($('#search-input').length) {
       searchmoveidx: (val, oldVal) => {
         if (val > 0) {
           vueHeader.searchmovekey = (vueHeader.searchmovearr[val - 1])
-            ? 'res.' + vueHeader.searchmovearr[val - 1]._id
+            ? 'res.' + vueHeader.searchmovearr[val - 1].entryPath
             : 'sug.' + vueHeader.searchmovearr[val - 1]
         } else {
           vueHeader.searchmovekey = ''
@@ -61,7 +61,7 @@ if ($('#search-input').length) {
         let i = vueHeader.searchmoveidx - 1
 
         if (vueHeader.searchmovearr[i]) {
-          window.location.assign('/' + vueHeader.searchmovearr[i]._id)
+          window.location.assign('/' + vueHeader.searchmovearr[i].entryPath)
         } else {
           vueHeader.searchq = vueHeader.searchmovearr[i]
         }

controllers/ws.js

@@ -13,7 +13,7 @@ module.exports = (socket) => {
 
   socket.on('search', (data, cb) => {
     cb = cb || _.noop
-    entries.search(data.terms).then((results) => {
+    search.find(data.terms).then((results) => {
       return cb(results) || true
     })
   })

libs/entries.js

@@ -256,7 +256,9 @@ module.exports = {
     return fs.statAsync(fpath).then((st) => {
       if (st.isFile()) {
         return self.makePersistent(entryPath, contents).then(() => {
-          return self.updateCache(entryPath)
+          return self.updateCache(entryPath).then(entry => {
+            return search.add(entry)
+          })
         })
       } else {
         return Promise.reject(new Error('Entry does not exist!'))

libs/git.js

@@ -68,13 +68,13 @@ module.exports = {
   _initRepo (appconfig) {
     let self = this
 
-    winston.info('[' + PROCNAME + '][GIT] Checking Git repository...')
+    winston.info('[' + PROCNAME + '.Git] Checking Git repository...')
 
     // -> Check if path is accessible
 
     return fs.mkdirAsync(self._repo.path).catch((err) => {
       if (err.code !== 'EEXIST') {
-        winston.error('[' + PROCNAME + '][GIT] Invalid Git repository path or missing permissions.')
+        winston.error('[' + PROCNAME + '.Git] Invalid Git repository path or missing permissions.')
       }
     }).then(() => {
       self._git = new Git({ 'git-dir': self._repo.path })
@@ -89,7 +89,7 @@ module.exports = {
       })
     }).then(() => {
       if (appconfig.git === false) {
-        winston.info('[' + PROCNAME + '][GIT] Remote syncing is disabled. Not recommended!')
+        winston.info('[' + PROCNAME + '.Git] Remote syncing is disabled. Not recommended!')
         return Promise.resolve(true)
       }
 
@@ -126,10 +126,10 @@ module.exports = {
         }
       })
     }).catch((err) => {
-      winston.error('[' + PROCNAME + '][GIT] Git remote error!')
+      winston.error('[' + PROCNAME + '.Git] Git remote error!')
       throw err
     }).then(() => {
-      winston.info('[' + PROCNAME + '][GIT] Git repository is OK.')
+      winston.info('[' + PROCNAME + '.Git] Git repository is OK.')
       return true
     })
   },
@@ -159,12 +159,12 @@ module.exports = {
 
     // Fetch
 
-    winston.info('[' + PROCNAME + '][GIT] Performing pull from remote repository...')
+    winston.info('[' + PROCNAME + '.Git] Performing pull from remote repository...')
     return self._git.pull('origin', self._repo.branch).then((cProc) => {
-      winston.info('[' + PROCNAME + '][GIT] Pull completed.')
+      winston.info('[' + PROCNAME + '.Git] Pull completed.')
     })
     .catch((err) => {
-      winston.error('[' + PROCNAME + '][GIT] Unable to fetch from git origin!')
+      winston.error('[' + PROCNAME + '.Git] Unable to fetch from git origin!')
       throw err
     })
     .then(() => {
@@ -174,12 +174,12 @@ module.exports = {
         let out = cProc.stdout.toString()
 
         if (_.includes(out, 'commit')) {
-          winston.info('[' + PROCNAME + '][GIT] Performing push to remote repository...')
+          winston.info('[' + PROCNAME + '.Git] Performing push to remote repository...')
           return self._git.push('origin', self._repo.branch).then(() => {
-            return winston.info('[' + PROCNAME + '][GIT] Push completed.')
+            return winston.info('[' + PROCNAME + '.Git] Push completed.')
           })
         } else {
-          winston.info('[' + PROCNAME + '][GIT] Push skipped. Repository is already in sync.')
+          winston.info('[' + PROCNAME + '.Git] Push skipped. Repository is already in sync.')
         }
 
         return true

libs/local.js

@@ -98,11 +98,12 @@ module.exports = {
    * @return     {Void}  Void
    */
   createBaseDirectories (appconfig) {
-    winston.info('[SERVER] Checking data directories...')
+    winston.info('[SERVER.Local] Checking data directories...')
 
     try {
       fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data))
       fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
+      fs.emptyDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
       fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './thumbs'))
       fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './temp-upload'))
 
@@ -120,7 +121,7 @@ module.exports = {
       winston.error(err)
     }
 
-    winston.info('[SERVER] Data and Repository directories are OK.')
+    winston.info('[SERVER.Local] Data and Repository directories are OK.')
 
     return
   },

libs/search.js

@@ -0,0 +1,206 @@
+'use strict'
+
+const Promise = require('bluebird')
+const _ = require('lodash')
+const path = require('path')
+const searchIndex = require('search-index')
+const stopWord = require('stopword')
+const streamToPromise = require('stream-to-promise')
+
+module.exports = {
+
+  _si: null,
+  _isReady: false,
+
+  /**
+   * Initialize search index
+   *
+   * @return {undefined} Void
+   */
+  init () {
+    let self = this
+    let dbPath = path.resolve(ROOTPATH, appconfig.paths.data, 'search')
+    self._isReady = new Promise((resolve, reject) => {
+      searchIndex({
+        deletable: true,
+        fieldedSearch: true,
+        indexPath: dbPath,
+        logLevel: 'error',
+        stopwords: _.get(stopWord, appconfig.lang, [])
+      }, (err, si) => {
+        if (err) {
+          winston.error('[SERVER.Search] Failed to initialize search index.', err)
+          reject(err)
+        } else {
+          self._si = Promise.promisifyAll(si)
+          self._si.flushAsync().then(() => {
+            winston.info('[SERVER.Search] Search index flushed and ready.')
+            resolve(true)
+          })
+        }
+      })
+    })
+
+    return self
+  },
+
+  /**
+   * Add a document to the index
+   *
+   * @param      {Object}   content  Document content
+   * @return     {Promise}  Promise of the add operation
+   */
+  add (content) {
+    let self = this
+
+    return self._isReady.then(() => {
+      return self.delete(content._id).then(() => {
+        return self._si.concurrentAddAsync({
+          fieldOptions: [{
+            fieldName: 'entryPath',
+            searchable: true,
+            weight: 2
+          },
+          {
+            fieldName: 'title',
+            nGramLength: [1, 2],
+            searchable: true,
+            weight: 3
+          },
+          {
+            fieldName: 'subtitle',
+            searchable: true,
+            weight: 1,
+            storeable: false
+          },
+          {
+            fieldName: 'parent',
+            searchable: false
+          },
+          {
+            fieldName: 'content',
+            searchable: true,
+            weight: 0,
+            storeable: false
+          }]
+        }, [{
+          entryPath: content._id,
+          title: content.title,
+          subtitle: content.subtitle || '',
+          parent: content.parent || '',
+          content: content.content || ''
+        }]).then(() => {
+          winston.log('verbose', '[SERVER.Search] Entry ' + content._id + ' added/updated to index.')
+          return true
+        }).catch((err) => {
+          winston.error(err)
+        })
+      }).catch((err) => {
+        winston.error(err)
+      })
+    })
+  },
+
+  /**
+   * Delete an entry from the index
+   *
+   * @param      {String}   The     entry path
+   * @return     {Promise}  Promise of the operation
+   */
+  delete (entryPath) {
+    let self = this
+
+    return self._isReady.then(() => {
+      return streamToPromise(self._si.search({
+        query: [{
+          AND: { 'entryPath': [entryPath] }
+        }]
+      })).then((results) => {
+        if (results.totalHits > 0) {
+          let delIds = _.map(results.hits, 'id')
+          return self._si.delAsync(delIds)
+        } else {
+          return true
+        }
+      }).catch((err) => {
+        if (err.type === 'NotFoundError') {
+          return true
+        } else {
+          winston.error(err)
+        }
+      })
+    })
+  },
+
+  /**
+   * Flush the index
+   *
+   * @returns {Promise} Promise of the flush operation
+   */
+  flush () {
+    let self = this
+    return self._isReady.then(() => {
+      return self._si.flushAsync()
+    })
+  },
+
+  /**
+   * Search the index
+   *
+   * @param {Array<String>} terms
+   * @returns {Promise<Object>} Hits and suggestions
+   */
+  find (terms) {
+    let self = this
+    terms = _.chain(terms)
+              .deburr()
+              .toLower()
+              .trim()
+              .replace(/[^a-z0-9 ]/g, '')
+              .value()
+    let arrTerms = _.chain(terms)
+                    .split(' ')
+                    .filter((f) => { return !_.isEmpty(f) })
+                    .value()
+
+    return streamToPromise(self._si.search({
+      query: [{
+        AND: { '*': arrTerms }
+      }],
+      pageSize: 10
+    })).then((hits) => {
+      if (hits.length > 0) {
+        hits = _.map(_.sortBy(hits, ['score']), h => {
+          return h.document
+        })
+      }
+      if (hits.length < 5) {
+        return streamToPromise(self._si.match({
+          beginsWith: terms,
+          threshold: 3,
+          limit: 5,
+          type: 'simple'
+        })).then((matches) => {
+          return {
+            match: hits,
+            suggest: matches
+          }
+        })
+      } else {
+        return {
+          match: hits,
+          suggest: []
+        }
+      }
+    }).catch((err) => {
+      if (err.type === 'NotFoundError') {
+        return {
+          match: [],
+          suggest: []
+        }
+      } else {
+        winston.error(err)
+      }
+    })
+  }
+}

package.json

@@ -91,6 +91,8 @@
     "simplemde": "^1.11.2",
     "socket.io": "^1.7.2",
     "sticky-js": "^1.0.7",
+    "stopword": "^0.1.1",
+    "stream-to-promise": "^2.2.0",
     "validator": "^6.2.0",
     "validator-as-promised": "^1.0.2",
     "winston": "^2.3.0"
@@ -139,7 +141,6 @@
       "app",
       "appconfig",
       "appdata",
-      "bgAgent",
       "db",
       "entries",
       "git",
@@ -147,6 +148,7 @@
       "lang",
       "lcdata",
       "rights",
+      "search",
       "upl",
       "winston",
       "ws",

server.js

@@ -37,6 +37,7 @@ global.entries = require('./libs/entries').init()
 global.git = require('./libs/git').init(false)
 global.lang = require('i18next')
 global.mark = require('./libs/markdown')
+global.search = require('./libs/search').init()
 global.upl = require('./libs/uploads').init()
 
 // ----------------------------------------
@@ -239,7 +240,19 @@ io.on('connection', ctrl.ws)
 // Start child processes
 // ----------------------------------------
 
-global.bgAgent = fork('agent.js')
+let bgAgent = fork('agent.js')
+
+bgAgent.on('message', m => {
+  if (!m.action) {
+    return
+  }
+
+  switch (m.action) {
+    case 'searchAdd':
+      search.add(m.content)
+      break
+  }
+})
 
 process.on('exit', (code) => {
   bgAgent.disconnect()

views/common/header.pug

@@ -26,8 +26,8 @@
       ul.searchresults-list
         li(v-if='searchres.length === 0')
           a: em No results matching your query
-        li(v-for='sres in searchres', v-bind:class='{ "is-active": searchmovekey === "res." + sres._id }')
-          a(v-bind:href='"/" + sres._id') {{ sres.title }}
+        li(v-for='sres in searchres', v-bind:class='{ "is-active": searchmovekey === "res." + sres.entryPath }')
+          a(v-bind:href='"/" + sres.entryPath') {{ sres.title }}
       p.searchresults-label(v-if='searchsuggest.length > 0') Did you mean...?
       ul.searchresults-list(v-if='searchsuggest.length > 0')
         li(v-for='sug in searchsuggest', v-bind:class='{ "is-active": searchmovekey === "sug." + sug }')