WebSocket server + Search index + indexable content parser

agent.js

@@ -7,10 +7,35 @@
 global.ROOTPATH = __dirname;
 
 // ----------------------------------------
-// Load modules
+// Load Winston
 // ----------------------------------------
 
+var _isDebug = process.env.NODE_ENV === 'development';
+
 global.winston = require('winston');
+winston.remove(winston.transports.Console)
+winston.add(winston.transports.Console, {
+  level: (_isDebug) ? 'info' : 'warn',
+  prettyPrint: true,
+  colorize: true,
+  silent: false,
+  timestamp: true
+});
+
+// ----------------------------------------
+// Fetch internal handshake key
+// ----------------------------------------
+
+if(!process.argv[2] || process.argv[2].length !== 40) {
+	winston.error('[WS] Illegal process start. Missing handshake key.');
+	process.exit(1);
+}
+global.WSInternalKey = process.argv[2];
+
+// ----------------------------------------
+// Load modules
+// ----------------------------------------
+
 winston.info('[AGENT] Background Agent is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
@@ -18,7 +43,6 @@ var appconfig = require('./models/config')('./config.yml');
 global.git = require('./models/git').init(appconfig);
 global.entries = require('./models/entries').init(appconfig);
 global.mark = require('./models/markdown');
-global.search = require('./models/search').init(appconfig);
 
 var _ = require('lodash');
 var moment = require('moment');
@@ -26,6 +50,8 @@ var Promise = require('bluebird');
 var fs = Promise.promisifyAll(require("fs-extra"));
 var path = require('path');
 var cron = require('cron').CronJob;
+var wsClient = require('socket.io-client');
+global.ws = wsClient('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
 
 // ----------------------------------------
 // Start Cron
@@ -90,8 +116,11 @@ var job = new cron({
 								//-> Update search index
 
 								if(fileStatus !== 'active') {
-									return entries.fetchTextVersion(entryPath).then((content) => {
-										console.log(content);
+									return entries.fetchIndexableVersion(entryPath).then((content) => {
+										ws.emit('searchAdd', {
+											auth: WSInternalKey,
+											content
+										});
 									});
 								}
 
@@ -114,25 +143,42 @@ var job = new cron({
 		// ----------------------------------------
 
 		Promise.all(jobs).then(() => {
-			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now... [' + moment().toISOString() + ']');
+			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now...');
 		}).catch((err) => {
-			winston.error('[AGENT] One or more jobs have failed [' + moment().toISOString() + ']: ', err);
+			winston.error('[AGENT] One or more jobs have failed: ', err);
 		}).finally(() => {
 			jobIsBusy = false;
 		});
 
 	},
-	start: true,
+	start: false,
 	timeZone: 'UTC',
 	runOnInit: true
 });
 
+// ----------------------------------------
+// Connect to local WebSocket server
+// ----------------------------------------
+
+ws.on('connect', function () {
+	job.start();
+	winston.info('[AGENT] Background Agent started successfully! [RUNNING]');
+});
+
+ws.on('connect_error', function () {
+	winston.warn('[AGENT] Unable to connect to WebSocket server! Retrying...');
+});
+ws.on('reconnect_failed', function () {
+	winston.error('[AGENT] Failed to reconnect to WebSocket server too many times! Stopping agent...');
+	process.exit(1);
+});
+
 // ----------------------------------------
 // Shutdown gracefully
 // ----------------------------------------
 
 process.on('disconnect', () => {
-	winston.warn('[AGENT] Lost connection to main server. Exiting... [' + moment().toISOString() + ']');
+	winston.warn('[AGENT] Lost connection to main server. Exiting...');
 	job.stop();
 	process.exit();
 });

client/js/app.js

@@ -52,6 +52,32 @@ jQuery( document ).ready(function( $ ) {
 
 	}
 
+	// ====================================
+	// Establish WebSocket connection
+	// ====================================
+
+	var socket = io(ioHost);
+
+	var vueHeader = new Vue({
+		el: '#header-container',
+		data: {
+			searchq: '',
+			searchres: []
+		},
+		watch: {
+			searchq: (val, oldVal) => {
+				if(val.length >= 3) {
+					socket.emit('search', { terms: val }, (data) => {
+						vueHeader.$set('searchres', data);
+					});
+				}
+			}
+		},
+		methods: {
+			
+		}
+	});
+
 	// ====================================
 	// Pages logic
 	// ====================================

client/scss/layout/_content.scss

@@ -1,4 +1,12 @@
 
+#page-type-view > section {
+	transition: background-color .5s ease;
+
+	&.blurred {
+		background-color: $grey-lighter;
+	}
+
+}
 
 .sd-menus {
 

client/scss/layout/_header.scss

@@ -25,4 +25,15 @@ h2.nav-item {
 		opacity: 1;
 	}
 
+}
+
+.searchresults {
+	position: fixed;
+	top: 45px;
+	left: 0;
+	right: 0;
+	margin: 0 auto;
+	width: 500px;
+	z-index: 1;
+	//display: none;
 }

config.sample.yml

@@ -1,38 +1,46 @@
-###################################################
-# REQUARKS WIKI - CONFIGURATION                   #
-###################################################
-# Full explanation + examples in the documentation (https://requarks-wiki.readme.io/)
+#######################################################################
+# REQUARKS WIKI - CONFIGURATION                                       #
+#######################################################################
+# Full explanation + examples in the documentation:
+# https://requarks-wiki.readme.io/
 
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 # Title of this site
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 
 title: Wiki
 
-# -------------------------------------------------
-# Full path to the site, without the trailing slash
-# -------------------------------------------------
+# ---------------------------------------------------------------------
+# Full public path to the site, without the trailing slash
+# ---------------------------------------------------------------------
 
 host: http://localhost
 
-# -------------------------------------------------
-# Port the server should listen to (80 by default)
-# -------------------------------------------------
+# ---------------------------------------------------------------------
+# Port the main server should listen to (80 by default)
+# ---------------------------------------------------------------------
 # To use process.env.PORT, comment the line below:
 
 port: 80
 
-# -------------------------------------------------
+# ---------------------------------------------------------------------
+# Port the websocket server should listen to (8080 by default)
+# ---------------------------------------------------------------------
+# Make sure this port is opened in the firewall if applicable
+
+wsPort: 8080
+
+# ---------------------------------------------------------------------
 # Data Directories
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 
 datadir:
   repo: ./repo
   db: ./data
 
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 # Git Connection Info
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 
 git:
   url: https://github.com/Organization/Repo
@@ -52,24 +60,25 @@ git:
     privateKey: /etc/requarkswiki/keys/git.key
     sslVerify: true
 
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 # Secret key to use when encrypting sessions
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 # Use a long and unique random string (256-bit keys are perfect!)
 
 sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz
 
-# -------------------------------------------------
+# ---------------------------------------------------------------------
 # Administrator email
-# -------------------------------------------------
-# An account will be created using the email specified here.
-# The password is set to "admin123" by default. Change it immediately upon login!!!
+# ---------------------------------------------------------------------
+# An admin account will be created using the email specified here.
+# The password is set to "admin123" by default. Change it immediately
+# upon login!!!
 
 admin: admin@company.com
 
-# -------------------------------------------------
-# Site UI Language
-# -------------------------------------------------
+# ---------------------------------------------------------------------
+# Site Language
+# ---------------------------------------------------------------------
 # Possible values: en, fr
 
 lang: en

gulpfile.js

@@ -19,6 +19,7 @@ var include = require("gulp-include");
  */
 var paths = {
 	scriptlibs: [
+		'./node_modules/socket.io-client/socket.io.js',
 		'./node_modules/jquery/dist/jquery.min.js',
 		'./node_modules/vue/dist/vue.min.js',
 		'./node_modules/jquery-smooth-scroll/jquery.smooth-scroll.min.js',

lib/internalAuth.js

@@ -0,0 +1,32 @@
+"use strict";
+
+const crypto = require('crypto');
+
+/**
+ * Internal Authentication
+ */
+module.exports = {
+
+	_curKey: false,
+
+	init(inKey) {
+
+		this._curKey = inKey;
+
+		return this;
+
+	},
+
+	generateKey() {
+
+		return crypto.randomBytes(20).toString('hex')
+
+	},
+
+	validateKey(inKey) {
+
+		return inKey === this._curKey;
+
+	}
+
+};

models/entries.js

@@ -183,7 +183,7 @@ module.exports = {
 	 * @param      {String}  entryPath  The entry path
 	 * @return     {String}  Text-only version
 	 */
-	fetchTextVersion(entryPath) {
+	fetchIndexableVersion(entryPath) {
 
 		let self = this;
 
@@ -192,11 +192,13 @@ module.exports = {
 			parseMeta: true,
 			parseTree: false,
 			includeMarkdown: true,
-			includeParentInfo: false,
+			includeParentInfo: true,
 			cache: false
 		}).then((pageData) => {
 			return {
+				entryPath,
 				meta: pageData.meta,
+				parent: pageData.parent || {},
 				text: mark.removeMarkdown(pageData.markdown)
 			};
 		});

models/git.js

@@ -160,7 +160,7 @@ module.exports = {
 
 				} else {
 
-					winston.info('[GIT] Repository is already in sync.');
+					winston.info('[GIT] Push skipped. Repository is already in sync.');
 
 				}
 

models/search.js

@@ -3,7 +3,7 @@
 var Promise = require('bluebird'),
 	_ = require('lodash'),
 	path = require('path'),
-	searchIndex = Promise.promisifyAll(require('search-index')),
+	searchIndex = require('search-index'),
 	stopWord = require('stopword');
 
 /**
@@ -21,9 +21,10 @@ module.exports = {
 	 */
 	init(appconfig) {
 
+		let self = this;
 		let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search-index');
 
-		this._si = searchIndex({
+		searchIndex({
 			deletable: true,
 			fieldedSearch: true,
 			indexPath: dbPath,
@@ -32,11 +33,86 @@ module.exports = {
 		}, (err, si) => {
 			if(err) {
 				winston.error('Failed to initialize search-index.', err);
+			} else {
+				self._si = Promise.promisifyAll(si);
 			}
 		});
 
-	}
+		return self;
+
+	},
+
+	find(terms) {
+
+		let self = this;
+		terms = _.chain(terms)
+							.deburr()
+							.toLower()
+							.trim()
+							.replace(/[^a-z0-9 ]/g, '')
+							.split(' ')
+							.filter((f) => { return !_.isEmpty(f); })
+							.value();
 
+		return self._si.searchAsync({
+			query: {
+				AND: [{ '*': terms }]
+			},
+			pageSize: 10
+		}).get('hits');
 
+	},
+
+	/**
+	 * Add a document to the index
+	 *
+	 * @param      {Object}   content  Document content
+	 * @return     {Promise}  Promise of the add operation
+	 */
+	add(content) {
+
+		let self = this;
+
+		return self._si.addAsync({
+			entryPath: content.entryPath,
+			title: content.meta.title,
+			subtitle: content.meta.subtitle || '',
+			parent: content.parent.title || '',
+			content: content.text || ''
+		}, {
+			fieldOptions: [{
+				fieldName: 'entryPath',
+				searchable: true,
+				weight: 2
+			},
+			{
+				fieldName: 'title',
+				nGramLength: [1, 2],
+				searchable: true,
+				weight: 3
+			},
+			{
+				fieldName: 'subtitle',
+				searchable: true,
+				weight: 1,
+				store: false
+			},
+			{
+				fieldName: 'subtitle',
+				searchable: false,
+			},
+			{
+				fieldName: 'content',
+				searchable: true,
+				weight: 0,
+				store: false
+			}]
+		}).then(() => {
+			winston.info('Entry ' + content.entryPath + ' added to index.');
+		}).catch((err) => {
+			winston.error(err);
+		});
+
+	}
 
 };

package.json

@@ -76,6 +76,7 @@
     "search-index": "^0.8.15",
     "serve-favicon": "^2.3.0",
     "simplemde": "^1.11.2",
+    "socket.io": "^1.4.8",
     "validator": "^5.5.0",
     "validator-as-promised": "^1.0.2",
     "winston": "^2.2.0"

server.js

@@ -10,7 +10,18 @@ global.ROOTPATH = __dirname;
 // Load global modules
 // ----------------------------------------
 
+var _isDebug = process.env.NODE_ENV === 'development';
+
 global.winston = require('winston');
+winston.remove(winston.transports.Console)
+winston.add(winston.transports.Console, {
+  level: (_isDebug) ? 'info' : 'warn',
+  prettyPrint: true,
+  colorize: true,
+  silent: false,
+  timestamp: true
+});
+
 winston.info('[SERVER] Requarks Wiki is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
@@ -161,8 +172,6 @@ app.use(function(err, req, res, next) {
 // Start HTTP server
 // ----------------------------------------
 
-winston.info('[SERVER] Requarks Wiki has initialized successfully.');
-
 winston.info('[SERVER] Starting HTTP server on port ' + appconfig.port + '...');
 
 app.set('port', appconfig.port);
@@ -193,12 +202,17 @@ server.on('listening', () => {
 });
 
 // ----------------------------------------
-// Start Agents
+// Start child processes
 // ----------------------------------------
 
-var fork = require('child_process').fork;
-var bgAgent = fork('agent.js');
+var fork = require('child_process').fork,
+    libInternalAuth = require('./lib/internalAuth'),
+    internalAuthKey = libInternalAuth.generateKey();
+
+var wsSrv = fork('ws-server.js', [internalAuthKey]),
+    bgAgent = fork('agent.js', [internalAuthKey]);
 
 process.on('exit', (code) => {
+  wsSrv.disconnect();
   bgAgent.disconnect();
 });

views/common/header.pug

@@ -1,34 +1,45 @@
 
-nav.nav.has-shadow.stickyscroll#header
-	.nav-left
-		block rootNavLeft
-			a.nav-item.is-brand(href='/')
-				img(src='/favicons/android-icon-96x96.png', alt='Wiki')
-			a.nav-item(href='/')
-				h1.title Wiki
-	.nav-center
-		block rootNavCenter
-			p.nav-item
-				input.input(type='text', placeholder='Search...', style= { 'max-width': '300px', width: '33vw' })
-	span.nav-toggle
-		span
-		span
-		span
-	.nav-right.nav-menu
-		block rootNavRight
-			i.nav-item#notifload
-			a.nav-item(href='/history/' + pageData.meta.path)
-				| History
-			a.nav-item(href='/source/' + pageData.meta.path)
-				| Source
-			span.nav-item
-				a.button(href='/edit/' + pageData.meta.path)
-					span.icon
-						i.fa.fa-edit
-					span Edit
-				a.button.is-primary.btn-create-prompt
-					span.icon
-						i.fa.fa-plus
-					span Create
+#header-container
+	nav.nav.has-shadow.stickyscroll#header
+		.nav-left
+			block rootNavLeft
+				a.nav-item.is-brand(href='/')
+					img(src='/favicons/android-icon-96x96.png', alt='Wiki')
+				a.nav-item(href='/')
+					h1.title Wiki
+		.nav-center
+			block rootNavCenter
+				p.nav-item
+					input.input(type='text', v-model='searchq', debounce='500' placeholder='Search...', style= { 'max-width': '300px', width: '33vw' })
+		span.nav-toggle
+			span
+			span
+			span
+		.nav-right.nav-menu
+			block rootNavRight
+				i.nav-item#notifload
+				a.nav-item(href='/history/' + pageData.meta.path)
+					| History
+				a.nav-item(href='/source/' + pageData.meta.path)
+					| Source
+				span.nav-item
+					a.button(href='/edit/' + pageData.meta.path)
+						span.icon
+							i.fa.fa-edit
+						span Edit
+					a.button.is-primary.btn-create-prompt
+						span.icon
+							i.fa.fa-plus
+						span Create
+
+	.box.searchresults
+		.menu
+			p.menu-label
+				| Search Results
+			ul.menu-list
+				li(v-for='sres in searchres')
+					a(href='#') {{ sres.document.title }}
+			p.menu-label
+				| Do you mean...?
 
 

views/layout.pug

@@ -24,6 +24,8 @@ html
 		// JS
 		script(type='text/javascript', src='/js/libs.js')
 		script(type='text/javascript', src='/js/app.js')
+		script(type='text/javascript').
+			var ioHost = window.location.origin + ':!{appconfig.wsPort}/';
 
 		block head
 

ws-server.js

@@ -0,0 +1,137 @@
+// ===========================================
+// REQUARKS WIKI - WebSocket Server
+// 1.0.0
+// Licensed under AGPLv3
+// ===========================================
+
+global.ROOTPATH = __dirname;
+
+// ----------------------------------------
+// Load Winston
+// ----------------------------------------
+
+var _isDebug = process.env.NODE_ENV === 'development';
+
+global.winston = require('winston');
+winston.remove(winston.transports.Console)
+winston.add(winston.transports.Console, {
+  level: (_isDebug) ? 'info' : 'warn',
+  prettyPrint: true,
+  colorize: true,
+  silent: false,
+  timestamp: true
+});
+
+// ----------------------------------------
+// Fetch internal handshake key
+// ----------------------------------------
+
+if(!process.argv[2] || process.argv[2].length !== 40) {
+	winston.error('[WS] Illegal process start. Missing handshake key.');
+	process.exit(1);
+}
+global.internalAuth = require('./lib/internalAuth').init(process.argv[2]);;
+
+// ----------------------------------------
+// Load modules
+// ----------------------------------------
+
+winston.info('[WS] WS Server is initializing...');
+
+var appconfig = require('./models/config')('./config.yml');
+
+global.entries = require('./models/entries').init(appconfig);
+global.mark = require('./models/markdown');
+global.search = require('./models/search').init(appconfig);
+
+// ----------------------------------------
+// Load modules
+// ----------------------------------------
+
+var _ = require('lodash');
+var express = require('express');
+var path = require('path');
+var http = require('http');
+var socketio = require('socket.io');
+var moment = require('moment');
+
+// ----------------------------------------
+// Define Express App
+// ----------------------------------------
+
+global.app = express();
+
+// ----------------------------------------
+// Controllers
+// ----------------------------------------
+
+app.get('/', function(req, res){
+  res.send('Requarks Wiki WebSocket server');
+});
+
+// ----------------------------------------
+// Start WebSocket server
+// ----------------------------------------
+
+winston.info('[SERVER] Starting WebSocket server on port ' + appconfig.wsPort + '...');
+
+app.set('port', appconfig.wsPort);
+var server = http.Server(app);
+var io = socketio(server);
+
+server.on('error', (error) => {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  switch (error.code) {
+    case 'EACCES':
+      console.error('Listening on port ' + appconfig.port + ' requires elevated privileges!');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error('Port ' + appconfig.port + ' is already in use!');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+});
+
+server.listen(appconfig.wsPort, () => {
+  winston.info('[WS] WebSocket server started successfully! [RUNNING]');
+});
+
+io.on('connection', (socket) => {
+
+	socket.on('searchAdd', (data) => {
+		if(internalAuth.validateKey(data.auth)) {
+			search.add(data.content);
+		}
+	});
+
+	socket.on('search', (data, cb) => {
+		search.find(data.terms).then((results) => {
+			cb(results);
+		});
+	})
+
+});
+
+/*setTimeout(() => {
+	search._si.searchAsync({ query: { AND: [{'*': ['unit']}] }}).then((stuff) => { console.log(stuff.hits); });
+}, 8000);*/
+
+// ----------------------------------------
+// Shutdown gracefully
+// ----------------------------------------
+
+process.on('disconnect', () => {
+	winston.warn('[WS] Lost connection to main server. Exiting... [' + moment().toISOString() + ']');
+	server.close();
+	process.exit();
+});
+
+process.on('exit', () => {
+	server.stop();
+});