feat: Started implementing generic parsing and caching of find queries

backend/package-lock.json

@@ -22,6 +22,7 @@
 				"mongoose": "^6.6.5",
 				"nodemailer": "^6.8.0",
 				"oauth": "^0.10.0",
+				"object-hash": "^3.0.0",
 				"redis": "^4.3.1",
 				"retry-axios": "^3.0.0",
 				"sha256": "^0.2.0",
@@ -33,6 +34,7 @@
 				"@microsoft/tsdoc": "^0.14.2",
 				"@types/async": "^3.2.15",
 				"@types/config": "^3.3.0",
+				"@types/object-hash": "^2.2.1",
 				"@typescript-eslint/eslint-plugin": "^5.40.0",
 				"@typescript-eslint/parser": "^5.40.0",
 				"eslint": "^8.25.0",
@@ -378,6 +380,12 @@
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.13.tgz",
 			"integrity": "sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw=="
 		},
+		"node_modules/@types/object-hash": {
+			"version": "2.2.1",
+			"resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-2.2.1.tgz",
+			"integrity": "sha512-i/rtaJFCsPljrZvP/akBqEwUP2y5cZLOmvO+JaYnz01aPknrQ+hB5MRcO7iqCUsFaYfTG8kGfKUyboA07xeDHQ==",
+			"dev": true
+		},
 		"node_modules/@types/strip-bom": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -3332,6 +3340,14 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/object-hash": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
+			"integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
+			"engines": {
+				"node": ">= 6"
+			}
+		},
 		"node_modules/object-inspect": {
 			"version": "1.12.2",
 			"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
@@ -4902,6 +4918,12 @@
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.13.tgz",
 			"integrity": "sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw=="
 		},
+		"@types/object-hash": {
+			"version": "2.2.1",
+			"resolved": "https://registry.npmjs.org/@types/object-hash/-/object-hash-2.2.1.tgz",
+			"integrity": "sha512-i/rtaJFCsPljrZvP/akBqEwUP2y5cZLOmvO+JaYnz01aPknrQ+hB5MRcO7iqCUsFaYfTG8kGfKUyboA07xeDHQ==",
+			"dev": true
+		},
 		"@types/strip-bom": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -7040,6 +7062,11 @@
 			"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
 			"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
 		},
+		"object-hash": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
+			"integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="
+		},
 		"object-inspect": {
 			"version": "1.12.2",
 			"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",

backend/package.json

@@ -29,6 +29,7 @@
 		"mongoose": "^6.6.5",
 		"nodemailer": "^6.8.0",
 		"oauth": "^0.10.0",
+		"object-hash": "^3.0.0",
 		"redis": "^4.3.1",
 		"retry-axios": "^3.0.0",
 		"sha256": "^0.2.0",
@@ -40,6 +41,7 @@
 		"@microsoft/tsdoc": "^0.14.2",
 		"@types/async": "^3.2.15",
 		"@types/config": "^3.3.0",
+		"@types/object-hash": "^2.2.1",
 		"@typescript-eslint/eslint-plugin": "^5.40.0",
 		"@typescript-eslint/parser": "^5.40.0",
 		"eslint": "^8.25.0",

backend/src/collections/abc.ts

@@ -7,7 +7,8 @@ export type AbcCollection = DefaultSchema & {
 		autofill: {
 			enabled: DocumentAttribute<{
 				type: BooleanConstructor;
-				required: true;
+				required: false;
+				restricted: true;
 			}>;
 		};
 	};
@@ -16,26 +17,31 @@ export type AbcCollection = DefaultSchema & {
 export const schema: AbcCollection = {
 	document: {
 		_id: {
-			type: mongoose.Schema.Types.ObjectId,
+			type: mongoose.Types.ObjectId,
 			required: true,
-			cacheKey: true
+			cacheKey: true,
+			restricted: false
 		},
 		createdAt: {
 			type: Date,
-			required: true
+			required: true,
+			restricted: false
 		},
 		updatedAt: {
 			type: Date,
-			required: true
+			required: true,
+			restricted: false
 		},
 		name: {
 			type: String,
-			required: true
+			required: true,
+			restricted: false
 		},
 		autofill: {
 			enabled: {
 				type: Boolean,
-				required: true // TODO: Set to false when fixed
+				required: false,
+				restricted: true // TODO: Set to false when empty 2nd layer object fixed
 			}
 		}
 	},

backend/src/modules/DataModule.ts

@@ -1,6 +1,7 @@
 import async from "async";
 import config from "config";
 import mongoose, { Schema } from "mongoose";
+import hash from "object-hash";
 import { createClient, RedisClientType } from "redis";
 import BaseModule from "../BaseModule";
 import ModuleManager from "../ModuleManager";
@@ -40,8 +41,10 @@ export default class DataModule extends BaseModule {
 
 					async () => {
 						if (this.collections) {
-							Object.values(this.collections).forEach(
-								collection => collection.model.syncIndexes()
+							await async.each(
+								Object.values(this.collections),
+								async collection =>
+									collection.model.syncIndexes()
 							);
 						} else
 							throw new Error("Collections have not been loaded");
@@ -168,15 +171,109 @@ export default class DataModule extends BaseModule {
 		});
 	}
 
-	// TODO decide on whether to throw an exception if no results found, possible configurable via param
+	// TODO split core into parseDocument(document, schema, { partial: boolean;  })
+	/**
+	 * parseQuery - Ensure validity of query
+	 *
+	 * @param query - Query
+	 * @param schema - Schema of collection document
+	 * @param options - Parser options
+	 * @returns Promise returning object with query values cast to schema types
+	 * 			and whether query includes restricted attributes
+	 */
+	private async parseQuery(
+		query: any,
+		schema: any,
+		options?: {
+			operators?: boolean;
+		}
+	): Promise<{ castQuery: any; restricted: boolean }> {
+		if (!query || typeof query !== "object")
+			throw new Error("Invalid query provided. Query must be an object.");
+		const keys = Object.keys(query);
+		if (keys.length === 0)
+			throw new Error("Invalid query provided. Query must contain keys.");
+		const operators = !(options && options.operators === false);
+		const castQuery: any = {};
+		let restricted = false;
+		await async.each(Object.entries(query), async ([key, value]) => {
+			if (operators && (key === "$or" || key === "$and")) {
+				if (!Array.isArray(value))
+					throw new Error(
+						`Key "${key}" must contain array of queries`
+					);
+				castQuery[key] = [];
+				await async.each(value, async _value => {
+					const { castQuery: _castQuery, restricted: _restricted } =
+						await this.parseQuery(_value, schema, options);
+					castQuery[key].push(_castQuery);
+					restricted = restricted || _restricted;
+				});
+			} else if (schema[key] !== undefined) {
+				if (schema[key].restricted) restricted = true;
+				if (
+					schema[key].type === undefined &&
+					Object.keys(schema[key]).length > 0
+				) {
+					const { castQuery: _castQuery, restricted: _restricted } =
+						await this.parseQuery(value, schema[key], options);
+					castQuery[key] = _castQuery;
+					restricted = restricted || _restricted;
+				} else if (
+					operators &&
+					typeof value === "object" &&
+					Object.keys(value).length === 1 &&
+					Array.isArray(value.$in)
+				) {
+					if (value.$in.length > 0)
+						castQuery[key] = {
+							$in: await async.map(
+								value.$in,
+								async (_value: any) => {
+									if (
+										key === "_id" &&
+										!schema[key].type.isValid(_value)
+									)
+										throw new Error(
+											"Invalid value for _id"
+										);
+									if (typeof schema[key].type === "function")
+										return new schema[key].type(_value);
+									throw new Error(
+										`Invalid schema type for ${key}`
+									);
+								}
+							)
+						};
+					else throw new Error(`Invalid value for ${key}`);
+				} else {
+					if (key === "_id" && !schema[key].type.isValid(value))
+						throw new Error("Invalid value for _id");
+					if (typeof schema[key].type === "function")
+						castQuery[key] = new schema[key].type(value);
+					else throw new Error(`Invalid schema type for ${key}`);
+				}
+			} else {
+				throw new Error(
+					`Invalid query provided. Key "${key}" not found`
+				);
+			}
+		});
+		return { castQuery, restricted };
+	}
+
 	// TODO hide sensitive fields
-	// TOOD don't store sensitive fields in cache
 	// TODO improve caching
 	// TODO add option to only request certain fields
 	// TODO add support for computed fields
-	// TODO parse query
+	// TODO parse query - validation
 	// TODO add proper typescript support
 	// TODO add proper jsdoc
+	// TODO add support for enum document attributes
+	// TODO add support for array document attributes
+	// TODO add support for reference document attributes
+	// TODO prevent caching if requiring restricted values
+	// TODO fix 2nd layer of schema
 	/**
 	 * find - Find data
 	 *
@@ -189,8 +286,7 @@ export default class DataModule extends BaseModule {
 		values, // TODO: Add support
 		limit = 1, // TODO have limit off by default?
 		page = 1,
-		useCache = true,
-		convertArrayToSingle = false
+		useCache = true
 	}: {
 		collection: T;
 		query: Record<string, any>;
@@ -198,11 +294,10 @@ export default class DataModule extends BaseModule {
 		limit?: number;
 		page?: number;
 		useCache?: boolean;
-		convertArrayToSingle?: boolean;
-	}): Promise<any> {
+	}): Promise<any | null> {
 		return new Promise((resolve, reject) => {
-			let addToCache = false;
-			let cacheKeyName: string | null = null;
+			let queryHash: string | null = null;
+			let cacheable = useCache !== false;
 
 			async.waterfall(
 				[
@@ -215,129 +310,100 @@ export default class DataModule extends BaseModule {
 					},
 
 					// Verify whether the query is valid-enough to continue
-					async () => {
-						if (
-							!query ||
-							typeof query !== "object" ||
-							Object.keys(query).length === 0
-						)
-							new Error(
-								"Invalid query provided. Query must be an object."
-							);
-					},
+					async () =>
+						this.parseQuery(
+							query,
+							this.collections![collection].schema.document
+						),
 
 					// If we can use cache, get from the cache, and if we get results return those, otherwise return null
-					async () => {
-						// Not using cache, so return
-						if (!useCache) return null;
-						// More than one query key, so impossible to get from cache
-						if (Object.keys(query).length > 1) return null;
-
-						// First key and only key in query object
-						const queryPropertyName = Object.keys(query)[0];
-						// Corresponding property from schema document
-						const documentProperty =
-							this.collections![collection].schema.document[
-								queryPropertyName
-							];
-
-						if (!documentProperty)
-							throw new Error(
-								`Query property ${queryPropertyName} not found in document.`
-							);
-						// If query name is not a cache key, just continue
-						if (!documentProperty.cacheKey) return null;
-
-						const values = [];
-						if (
-							Object.prototype.hasOwnProperty.call(
-								query[queryPropertyName],
-								"$in"
-							)
-						)
-							values.push(...query[queryPropertyName].$in);
-						else values.push(query[queryPropertyName]);
-
-						const cachedDocuments: any[] = [];
-
-						await async.each(values, async value =>
-							this.redis
-								?.GET(
-									`${collection}.${queryPropertyName}.${value.toString()}`
-								)
-								.then((cachedDocument: any) => {
-									if (cachedDocument)
-										cachedDocuments.push(
-											JSON.parse(cachedDocument)
-										);
-								})
+					async ({ castQuery, restricted }: any) => {
+						// Not using cache or query contains restricted values, so return
+						if (!cacheable || restricted)
+							return { castQuery, cachedDocuments: null };
+						queryHash = hash(
+							{ collection, castQuery, values, limit, page },
+							{
+								algorithm: "sha1"
+							}
 						);
-
-						// TODO optimize this
-						if (cachedDocuments.length !== values.length) {
-							addToCache = true;
-							cacheKeyName = queryPropertyName;
-							return null;
-						}
-
-						return cachedDocuments;
+						const cachedQuery = await this.redis?.GET(
+							`query.find.${queryHash}`
+						);
+						return {
+							castQuery,
+							cachedDocuments: cachedQuery
+								? JSON.parse(cachedQuery)
+								: null
+						};
 					},
 
 					// If we didn't get documents from the cache, get them from mongo
-					async (cachedDocuments: any[] | null) => {
-						if (cachedDocuments) return cachedDocuments;
-
+					async ({ castQuery, cachedDocuments }: any) => {
+						if (cachedDocuments) {
+							cacheable = false;
+							return cachedDocuments;
+						}
+						const getFindValues = async (object: any) => {
+							const find: any = {};
+							await async.each(
+								Object.entries(object),
+								async ([key, value]) => {
+									if (
+										value.type === undefined &&
+										Object.keys(value).length > 0
+									) {
+										const _find = await getFindValues(
+											value
+										);
+										if (Object.keys(_find).length > 0)
+											find[key] = _find;
+									} else if (!value.restricted)
+										find[key] = true;
+								}
+							);
+							return find;
+						};
+						const find: any = await getFindValues(
+							this.collections![collection].schema.document
+						);
 						return this.collections?.[collection].model
-							.find(query)
+							.find(castQuery, find)
 							.limit(limit)
 							.skip((page - 1) * limit);
 					},
 
-					// Convert documents from Mongoose model to regular objects, and if we got no documents throw an error
-					async (documents: any[]) => {
-						if (documents.length === 0)
-							throw new Error("No results found.");
-
-						return documents.map(document => {
-							if (!document._doc) return document;
-
-							const rawDocument = document._doc;
-							rawDocument._id = rawDocument._id.toString();
-							return rawDocument;
-						});
-					},
+					// Convert documents from Mongoose model to regular objects
+					async (documents: any[]) =>
+						async.map(documents, async (document: any) => {
+							const { castQuery } = await this.parseQuery(
+								document._doc || document,
+								this.collections![collection].schema.document,
+								{ operators: false }
+							);
+							return castQuery;
+						}),
 
 					// Add documents to the cache
 					async (documents: any[]) => {
-						// TODO only add new things to cache
-						// Adds the fetched documents to the cache, but doesn't wait for it to complete
-						if (addToCache && cacheKeyName) {
-							async.each(
-								documents,
-								// TODO verify that the cache key name property actually exists for these documents
-								async (document: any) =>
-									this.redis!.SET(
-										`${collection}.${cacheKeyName}.${document[
-											cacheKeyName!
-										].toString()}`,
-										JSON.stringify(document),
-										{
-											EX: 60
-										}
-									)
+						// Adds query results to cache but doesnt await
+						if (cacheable && queryHash) {
+							this.redis!.SET(
+								`query.find.${queryHash}`,
+								JSON.stringify(documents),
+								{
+									EX: 60
+								}
 							);
 						}
-
 						return documents;
 					}
 				],
 				(err, documents?: any[]) => {
 					if (err) reject(err);
-					else if (convertArrayToSingle)
-						resolve(
-							documents!.length === 1 ? documents![0] : documents
-						);
-					else resolve(documents);
+					else if (!documents || documents!.length === 0)
+						resolve(limit === 1 ? null : []);
+					else resolve(limit === 1 ? documents![0] : documents);
 				}
 			);
 		});

backend/src/types/Collections.ts

@@ -5,11 +5,14 @@ export type DocumentAttribute<
 	T extends {
 		type: unknown;
 		required?: boolean;
+		cacheKey?: boolean;
+		restricted?: boolean;
 	}
 > = {
 	type: T["type"];
-	required: T extends { required: false } ? false : true;
-	cacheKey?: boolean;
+	required: T["required"]; // TODO fix default unknown
+	cacheKey?: T["cacheKey"]; // TODO fix default unknown
+	restricted: T["restricted"]; // TODO fix default unknown
 };
 
 export type DefaultSchema = {
@@ -19,7 +22,7 @@ export type DefaultSchema = {
 		| Record<string, DocumentAttribute<{ type: unknown }>>
 	> & {
 		_id: DocumentAttribute<{
-			type: typeof mongoose.Schema.Types.ObjectId;
+			type: typeof mongoose.Types.ObjectId;
 		}>;
 		createdAt: DocumentAttribute<{ type: DateConstructor }>;
 		updatedAt: DocumentAttribute<{ type: DateConstructor }>;