refactor: Started adding basis of DataModule

backend/package-lock.json

@@ -31,6 +31,7 @@
       },
       "devDependencies": {
         "@types/async": "^3.2.15",
+        "@types/config": "^3.3.0",
         "@typescript-eslint/eslint-plugin": "^5.40.0",
         "@typescript-eslint/parser": "^5.40.0",
         "eslint": "^8.25.0",
@@ -336,6 +337,12 @@
       "integrity": "sha512-PAmPfzvFA31mRoqZyTVsgJMsvbynR429UTTxhmfsUCrWGh3/fxOrzqBtaTPJsn4UtzTv4Vb0+/O7CARWb69N4g==",
       "dev": true
     },
+    "node_modules/@types/config": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/@types/config/-/config-3.3.0.tgz",
+      "integrity": "sha512-9kZSbl3/X3TVNowLCu5HFQdQmD+4287Om55avknEYkuo6R2dDrsp/EXEHUFvfYeG7m1eJ0WYGj+cbcUIhARJAQ==",
+      "dev": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.11",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",
@@ -4911,6 +4918,12 @@
       "integrity": "sha512-PAmPfzvFA31mRoqZyTVsgJMsvbynR429UTTxhmfsUCrWGh3/fxOrzqBtaTPJsn4UtzTv4Vb0+/O7CARWb69N4g==",
       "dev": true
     },
+    "@types/config": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/@types/config/-/config-3.3.0.tgz",
+      "integrity": "sha512-9kZSbl3/X3TVNowLCu5HFQdQmD+4287Om55avknEYkuo6R2dDrsp/EXEHUFvfYeG7m1eJ0WYGj+cbcUIhARJAQ==",
+      "dev": true
+    },
     "@types/json-schema": {
       "version": "7.0.11",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",

backend/package.json

@@ -38,6 +38,7 @@
   },
   "devDependencies": {
     "@types/async": "^3.2.15",
+    "@types/config": "^3.3.0",
     "@typescript-eslint/eslint-plugin": "^5.40.0",
     "@typescript-eslint/parser": "^5.40.0",
     "eslint": "^8.25.0",

backend/src/BaseModule.ts

@@ -1,5 +1,5 @@
 import ModuleManager from "./ModuleManager";
-import { ModuleStatus } from "./types/ModuleStatus";
+import { ModuleStatus } from "./types/Modules";
 
 // type ModuleName = keyof Modules;
 

backend/src/ModuleManager.ts

@@ -2,13 +2,7 @@ import async from "async";
 import BaseModule from "./BaseModule";
 import Job from "./Job";
 import JobQueue from "./JobQueue";
-import { ModuleStatus } from "./types/ModuleStatus";
-import { Jobs, Modules } from "./types/TestModules";
-
-type ModuleClass<Module extends typeof BaseModule> = {
-	// eslint-disable-next-line
-	new (moduleManager: ModuleManager): Module;
-};
+import { Jobs, Modules, ModuleStatus, ModuleClass } from "./types/Modules";
 
 export default class ModuleManager {
 	private modules?: Modules;
@@ -180,24 +174,19 @@ export default class ModuleManager {
 	 * @param {[ any, { priority?: number }? ]} params Params
 	 */
 	public runJob<
-		ModuleName extends keyof Jobs & keyof Modules,
-		JobName extends keyof Jobs[ModuleName] &
-			keyof Omit<Modules[ModuleName], keyof BaseModule>,
-		Payload extends "payload" extends keyof Jobs[ModuleName][JobName]
-			? Jobs[ModuleName][JobName]["payload"]
+		M extends keyof Jobs & keyof Modules,
+		J extends keyof Jobs[M] & keyof Omit<Modules[M], keyof BaseModule>,
+		P extends "payload" extends keyof Jobs[M][J]
+			? Jobs[M][J]["payload"]
 			: undefined,
-		ReturnType = "returns" extends keyof Jobs[ModuleName][JobName]
-			? Jobs[ModuleName][JobName]["returns"]
-			: never
+		R = "returns" extends keyof Jobs[M][J] ? Jobs[M][J]["returns"] : never
 	>(
-		moduleName: ModuleName,
-		jobName: JobName,
-		...params: Payload extends undefined
-			? []
-			: [Payload, { priority?: number }?]
-	): Promise<ReturnType> {
+		moduleName: M,
+		jobName: J,
+		...params: P extends undefined ? [] : [P, { priority?: number }?]
+	): Promise<R> {
 		const [payload, options] = params;
-		return new Promise<ReturnType>((resolve, reject) => {
+		return new Promise<R>((resolve, reject) => {
 			const module = this.modules && this.modules[moduleName];
 			if (!module) reject(new Error("Module not found."));
 			else {
@@ -213,15 +202,16 @@ export default class ModuleManager {
 						new Job(
 							jobName.toString(),
 							moduleName,
-							async (resolveJob, rejectJob) => {
-								jobFunction(payload)
-									.then((response: ReturnType) => {
+							(resolveJob, rejectJob) => {
+								jobFunction
+									.apply(module, [payload])
+									.then((response: R) => {
 										resolveJob();
 										resolve(response);
 									})
-									.catch(() => {
+									.catch((err: any) => {
 										rejectJob();
-										reject();
+										reject(err);
 									});
 							},
 							{

backend/src/collections/abc.ts

@@ -0,0 +1,43 @@
+import mongoose from "mongoose";
+import { DocumentAttribute, DefaultSchema } from "../types/Collections";
+
+export type AbcCollection = DefaultSchema & {
+	document: {
+		name: DocumentAttribute<{ type: StringConstructor }>;
+		autofill: {
+			enabled: DocumentAttribute<{
+				type: BooleanConstructor;
+				required: true;
+			}>;
+		};
+	};
+};
+
+export const schema: AbcCollection = {
+	document: {
+		_id: {
+			type: mongoose.Schema.Types.ObjectId,
+			required: true
+		},
+		createdAt: {
+			type: Date,
+			required: true
+		},
+		updatedAt: {
+			type: Date,
+			required: true
+		},
+		name: {
+			type: String,
+			required: true
+		},
+		autofill: {
+			enabled: {
+				type: Boolean,
+				required: true // TODO: Set to false when fixed
+			}
+		}
+	},
+	timestamps: true,
+	version: 1
+};

backend/src/modules/DataModule.ts

@@ -0,0 +1,311 @@
+import async from "async";
+import config from "config";
+import mongoose, { Schema } from "mongoose";
+import { createClient, RedisClientType } from "redis";
+import BaseModule from "../BaseModule";
+import ModuleManager from "../ModuleManager";
+import { UniqueMethods } from "../types/Modules";
+import { Collections } from "../types/Collections";
+
+export default class DataModule extends BaseModule {
+	collections?: Collections;
+
+	redis?: RedisClientType;
+
+	/**
+	 * Data Module
+	 *
+	 * @param {ModuleManager} moduleManager Module manager class
+	 */
+	public constructor(moduleManager: ModuleManager) {
+		super(moduleManager, "data");
+	}
+
+	/**
+	 * startup - Startup data module
+	 */
+	public override startup(): Promise<void> {
+		return new Promise((resolve, reject) => {
+			async.waterfall(
+				[
+					(next: any) => {
+						super
+							.startup()
+							.then(() => next())
+							.catch(next);
+					},
+
+					(next: any) => {
+						const mongoUrl = config.get<any>("mongo").url;
+						mongoose
+							.connect(mongoUrl)
+							.then(() => {
+								this.loadCollections().then(() => {
+									if (this.collections) {
+										Object.values(this.collections).forEach(
+											collection =>
+												collection.model.syncIndexes()
+										);
+										next();
+									} else
+										next(
+											new Error(
+												"Collections have not been loaded"
+											)
+										);
+								});
+							})
+							.catch(next);
+					},
+
+					(next: any) => {
+						const { url, password } = config.get<any>("redis");
+						this.redis = createClient({
+							url,
+							password
+						});
+						this.redis
+							.connect()
+							.then(() => next())
+							.catch(next);
+					},
+
+					(next: any) => {
+						if (this.redis)
+							this.redis
+								.sendCommand([
+									"CONFIG",
+									"GET",
+									"notify-keyspace-events"
+								])
+								.then(res => {
+									if (
+										!(Array.isArray(res) && res[1] === "xE")
+									)
+										next(
+											new Error(
+												`notify-keyspace-events is NOT configured correctly! It is set to: ${
+													(Array.isArray(res) &&
+														res[1]) ||
+													"unknown"
+												}`
+											)
+										);
+									else next();
+								})
+								.catch(next);
+						else
+							next(new Error("Redis connection not established"));
+					},
+
+					(next: any) => {
+						super.started();
+						next();
+					}
+				],
+				err => {
+					if (err) reject(err);
+					else resolve();
+				}
+			);
+		});
+	}
+
+	/**
+	 * shutdown - Shutdown data module
+	 */
+	public override shutdown(): Promise<void> {
+		return new Promise(resolve => {
+			super
+				.shutdown()
+				.then(() => {
+					// TODO: Ensure the following shutdown correctly
+					if (this.redis) this.redis.disconnect();
+					mongoose.connection.close(false);
+				})
+				.finally(() => resolve());
+		});
+	}
+
+	/**
+	 * loadColllection - Import and load collection schema
+	 *
+	 * @param {string} collectionName Name of the collection
+	 * @returns {Collections[T]} Collection
+	 */
+	private loadCollection<T extends keyof Collections>(
+		collectionName: T
+	): Promise<Collections[T]> {
+		return new Promise(resolve => {
+			import(`../collections/${collectionName.toString()}`).then(
+				({ schema }: { schema: Collections[T]["schema"] }) => {
+					const mongoSchema = new Schema<
+						Collections[T]["schema"]["document"]
+					>(schema.document, {
+						timestamps: schema.timestamps
+					});
+					const model = mongoose.model(
+						collectionName.toString(),
+						mongoSchema
+					);
+					// @ts-ignore
+					resolve({
+						// @ts-ignore
+						schema,
+						// @ts-ignore
+						model
+					});
+				}
+			);
+		});
+	}
+
+	/**
+	 * loadCollections - Load and initialize all collections
+	 *
+	 * @returns {Promise} Promise
+	 */
+	private loadCollections(): Promise<void> {
+		return new Promise((resolve, reject) => {
+			const fetchCollections = async () => ({
+				abc: await this.loadCollection("abc")
+			});
+			fetchCollections()
+				.then(collections => {
+					this.collections = collections;
+					resolve();
+				})
+				.catch(err => {
+					reject(new Error(err));
+				});
+		});
+	}
+
+	/**
+	 * find - Find data
+	 *
+	 * @param {object} payload Payload
+	 * @param {string} payload.collection Name of collection to fetch from
+	 * @param {object} payload.query Query
+	 * @param {object} payload.values Return specific values
+	 * @param {number} payload.limit Returned data limit
+	 * @param {number} payload.cache Cache expiry in seconds (-1 to disable)
+	 * @returns {Promise} Return object
+	 */
+	public find<T extends keyof Collections>({
+		collection,
+		query,
+		values, // TODO: Add support
+		limit = 1, // TODO: Add pagination
+		cache = 60
+	}: {
+		collection: T;
+		query: Record<string, any>;
+		values?: Record<string, any>;
+		limit?: number;
+		cache?: number;
+	}): Promise<any> {
+		return new Promise((resolve, reject) => {
+			if (
+				this.redis &&
+				this.collections &&
+				this.collections[collection]
+			) {
+				async.waterfall(
+					[
+						(next: any) => {
+							const idsProvided: any = []; // TODO: Handle properly (e.g. one missing $in causes duplicate or many queries with mixed/no _id)
+							(
+								(query._id && query._id.$in) || [query._id]
+							).forEach((queryId: any) =>
+								idsProvided.push(queryId.toString())
+							);
+							const cached: any = [];
+							if (cache === -1 || idsProvided.length === 0)
+								next(null, cached, idsProvided);
+							else {
+								async.each(
+									idsProvided,
+									(queryId, _next) => {
+										this.redis
+											?.GET(`${collection}.${queryId}`)
+											.then((cacheValue: any) => {
+												if (cacheValue)
+													cached.push(
+														JSON.parse(cacheValue) // TODO: Convert _id to ObjectId
+													);
+												_next();
+											})
+											.catch(_next);
+									},
+									err => next(err, cached, idsProvided)
+								);
+							}
+						},
+
+						(cached: any, idsProvided: any, next: any) => {
+							if (idsProvided.length === cached.length)
+								next(null, [], cached);
+							else
+								this.collections?.[collection].model
+									.find(query)
+									.limit(limit)
+									.exec((err: any, res: any) => {
+										if (
+											err ||
+											(res.length === 0 &&
+												cached.length === 0)
+										)
+											next(
+												new Error(
+													err || "No results found"
+												)
+											);
+										else {
+											next(null, res, cached);
+										}
+									});
+						},
+
+						(response: any, cached: any, next: any) => {
+							if (cache > -1 && response.length > 0)
+								async.each(
+									response,
+									(res: any, _next) => {
+										this.redis
+											?.SET(
+												`${collection}.${res._id.toString()}`,
+												JSON.stringify(res)
+											)
+											.then(() => {
+												this.redis
+													?.EXPIRE(
+														`${collection}.${res._id.toString()}`,
+														cache
+													)
+													.then(() => _next())
+													.catch(_next);
+											})
+											.catch(_next);
+									},
+									err => next(err, [...response, ...cached])
+								);
+							else next(null, [...response, ...cached]);
+						}
+					],
+					(err, res: any) => {
+						if (err) reject(err);
+						else resolve(res.length === 1 ? res[0] : res);
+					}
+				);
+			} else reject(new Error(`Collection "${collection}" not loaded`));
+		});
+	}
+}
+
+export type DataModuleJobs = {
+	[Property in keyof UniqueMethods<DataModule>]: {
+		payload: Parameters<UniqueMethods<DataModule>[Property]>[0];
+		returns: Awaited<ReturnType<UniqueMethods<DataModule>[Property]>>;
+	};
+};

backend/src/modules/OtherModule.ts

@@ -1,4 +1,4 @@
-import { UniqueMethods } from "../types/TestModules";
+import { UniqueMethods } from "../types/Modules";
 import BaseModule from "../BaseModule";
 import ModuleManager from "../ModuleManager";
 

backend/src/modules/StationModule.ts

@@ -1,4 +1,4 @@
-import { UniqueMethods } from "../types/TestModules";
+import { UniqueMethods } from "../types/Modules";
 import BaseModule from "../BaseModule";
 import ModuleManager from "../ModuleManager";
 

backend/src/types/Collections.ts

@@ -0,0 +1,35 @@
+import mongoose, { Model } from "mongoose";
+import { AbcCollection } from "../collections/abc";
+
+export type DocumentAttribute<
+	T extends {
+		type: unknown;
+		required?: boolean;
+	}
+> = {
+	type: T["type"];
+	required: T extends { required: false } ? false : true;
+};
+
+export type DefaultSchema = {
+	document: Record<
+		string,
+		| DocumentAttribute<{ type: unknown }>
+		| Record<string, DocumentAttribute<{ type: unknown }>>
+	> & {
+		_id: DocumentAttribute<{
+			type: typeof mongoose.Schema.Types.ObjectId;
+		}>;
+		createdAt: DocumentAttribute<{ type: DateConstructor }>;
+		updatedAt: DocumentAttribute<{ type: DateConstructor }>;
+	};
+	timestamps: boolean;
+	version: number;
+};
+
+export type Collections = {
+	abc: {
+		schema: AbcCollection;
+		model: Model<AbcCollection["document"]>;
+	};
+};

backend/src/types/ModuleClass.ts

@@ -1,7 +0,0 @@
-import ModuleManager from "../ModuleManager";
-import { ValueOf } from "./ValueOf";
-import { Modules } from "./TestModules";
-
-export type ModuleClass = {
-	new (moduleManager: ModuleManager): ValueOf<Modules>;
-};

backend/src/types/ModuleStatus.ts

@@ -1,8 +0,0 @@
-export type ModuleStatus =
-	| "LOADED"
-	| "STARTING"
-	| "STARTED"
-	| "STOPPED"
-	| "STOPPING"
-	| "ERROR"
-	| "DISABLED";

backend/src/types/Modules.ts

@@ -0,0 +1,44 @@
+import DataModule, { DataModuleJobs } from "../modules/DataModule";
+import OtherModule, { OtherModuleJobs } from "../modules/OtherModule";
+import StationModule, { StationModuleJobs } from "../modules/StationModule";
+import ModuleManager from "../ModuleManager";
+import BaseModule from "../BaseModule";
+
+export type ModuleClass<Module extends typeof BaseModule> = {
+	new (moduleManager: ModuleManager): Module;
+};
+
+export type Jobs = {
+	data: {
+		[Property in keyof DataModuleJobs]: DataModuleJobs[Property];
+	};
+	others: {
+		[Property in keyof OtherModuleJobs]: OtherModuleJobs[Property];
+	};
+	stations: {
+		[Property in keyof StationModuleJobs]: StationModuleJobs[Property];
+	};
+};
+
+export type Modules = {
+	data: DataModule & typeof BaseModule;
+	others: OtherModule & typeof BaseModule;
+	stations: StationModule & typeof BaseModule;
+};
+
+export type ModuleStatus =
+	| "LOADED"
+	| "STARTING"
+	| "STARTED"
+	| "STOPPED"
+	| "STOPPING"
+	| "ERROR"
+	| "DISABLED";
+
+export type Methods<T> = {
+	[P in keyof T as T[P] extends (...args: any) => Awaited<any>
+		? P
+		: never]: T[P];
+};
+
+export type UniqueMethods<T> = Methods<Omit<T, keyof BaseModule>>;

backend/src/types/TestModules.ts

@@ -1,23 +0,0 @@
-import BaseModule from "../BaseModule";
-import OtherModule, { OtherModuleJobs } from "../modules/OtherModule";
-import StationModule, { StationModuleJobs } from "../modules/StationModule";
-
-export type Methods<T> = {
-	[P in keyof T as T[P] extends Function ? P : never]: T[P];
-};
-
-export type UniqueMethods<T> = Methods<Omit<T, keyof typeof BaseModule>>;
-
-export type Jobs = {
-	others: {
-		[Property in keyof OtherModuleJobs]: OtherModuleJobs[Property];
-	};
-	stations: {
-		[Property in keyof StationModuleJobs]: StationModuleJobs[Property];
-	};
-};
-
-export type Modules = {
-	others: OtherModule & typeof BaseModule;
-	stations: StationModule & typeof BaseModule;
-};

backend/src/types/ValueOf.ts

@@ -1 +0,0 @@
-export type ValueOf<T> = T[keyof T];