soajs.controller 4.3.3 → 4.3.5

package/mw/idempotency/index.js

@@ -0,0 +1,171 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
+const memory = require('./model/memory.js');
+const mongo = require('./model/mongo.js');
+
+let model = { memory, mongo };
+
+const UUID_V4_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+function isValidUUIDv4(str) {
+	return UUID_V4_REGEX.test(str);
+}
+
+function matchesApi(method, path, apis) {
+	if (!apis || !Array.isArray(apis)) {
+		return false;
+	}
+	const requestSignature = `${method} ${path}`;
+	for (let api of apis) {
+		if (api === requestSignature) {
+			return true;
+		}
+		const [apiMethod, apiPath] = api.split(' ');
+		if (apiMethod !== method) {
+			continue;
+		}
+		const apiParts = apiPath.split('/');
+		const pathParts = path.split('/');
+		if (apiParts.length !== pathParts.length) {
+			continue;
+		}
+		let matches = true;
+		for (let i = 0; i < apiParts.length; i++) {
+			if (apiParts[i].startsWith(':')) {
+				continue;
+			}
+			if (apiParts[i] !== pathParts[i]) {
+				matches = false;
+				break;
+			}
+		}
+		if (matches) {
+			return true;
+		}
+	}
+	return false;
+}
+
+module.exports = function (configuration) {
+	model.memory.init(configuration.log);
+	model.mongo.init(configuration.log, configuration.gatewayDB);
+
+	return (req, res, next) => {
+		const idempotencyKey = req.headers['idempotency-key'];
+
+		if (!idempotencyKey) {
+			return next();
+		}
+
+		if (!isValidUUIDv4(idempotencyKey)) {
+			req.soajs.controllerResponse({
+				'status': 400,
+				'code': 180,
+				'msg': 'Invalid Idempotency-Key format. Expected UUID v4.'
+			});
+			return;
+		}
+
+		if (!req.soajs || !req.soajs.tenant || !req.soajs.controller || !req.soajs.controller.serviceParams) {
+			return next();
+		}
+
+		const serviceName = req.soajs.controller.serviceParams.name;
+		// Strip service prefix from pathname to get API path (e.g., /console/registry -> /registry)
+		const serviceInfo = req.soajs.controller.serviceParams.serviceInfo;
+		const apiPath = "/" + serviceInfo.slice(2).join("/");
+		const method = req.method.toUpperCase();
+
+		if (method === 'GET') {
+			return next();
+		}
+
+		const idempotencyConfig = get(["soajs", "registry", "custom", "gateway", "value", "idempotency"], req);
+		if (!idempotencyConfig) {
+			return next();
+		}
+
+		const serviceConfig = idempotencyConfig.services && idempotencyConfig.services[serviceName];
+		if (!serviceConfig || !serviceConfig.enabled) {
+			return next();
+		}
+
+		if (serviceConfig.apis && !matchesApi(method, apiPath, serviceConfig.apis)) {
+			return next();
+		}
+
+		const modelType = idempotencyConfig.model || 'memory';
+		if (!model[modelType]) {
+			configuration.log.warn('Idempotency: Unknown model [' + modelType + ']. It can only be [memory || mongo]');
+			return next();
+		}
+
+		const ttl = serviceConfig.ttl || 60000;
+		const key = {
+			'l1': req.soajs.tenant.id,
+			'l2': idempotencyKey
+		};
+
+		const processIdempotency = async () => {
+			try {
+				const existing = await model[modelType].get(key);
+
+				if (existing) {
+					if (existing.status === 'in_flight') {
+						req.soajs.controllerResponse({
+							'status': 409,
+							'code': 181,
+							'msg': 'Request with this Idempotency-Key is still being processed.'
+						});
+						return;
+					}
+
+					if (existing.status === 'completed' && existing.response) {
+						res.writeHead(existing.response.statusCode, existing.response.headers);
+						res.end(existing.response.body);
+						return;
+					}
+				}
+
+				const locked = await model[modelType].lock(key, ttl);
+				if (!locked) {
+					req.soajs.controllerResponse({
+						'status': 409,
+						'code': 181,
+						'msg': 'Request with this Idempotency-Key is still being processed.'
+					});
+					return;
+				}
+
+				// Store idempotency context for gotoService to use
+				req.soajs.idempotencyContext = {
+					key: key,
+					ttl: ttl,
+					modelType: modelType,
+					model: model[modelType],
+					log: configuration.log
+				};
+
+				next();
+			} catch (err) {
+				configuration.log.error('Idempotency error:', err);
+				model[modelType].unlock(key).catch(() => {});
+				next();
+			}
+		};
+
+		processIdempotency();
+	};
+};
+
+// Export model for external use
+module.exports.model = model;

package/mw/idempotency/model/memory.js

@@ -0,0 +1,128 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+let dataHolder = {};
+let cleanupInterval = null;
+
+let model = {
+	'log': null,
+
+	'init': (log) => {
+		model.log = log;
+		dataHolder = {};
+
+		if (cleanupInterval) {
+			clearInterval(cleanupInterval);
+		}
+		const CLEANUP_INTERVAL = process.env.SOAJS_IDEMPOTENCY_CLEANUP_INTERVAL || 60000;
+		cleanupInterval = setInterval(() => {
+			model.cleanupExpiredEntries();
+		}, CLEANUP_INTERVAL);
+	},
+
+	'cleanupExpiredEntries': () => {
+		const now = Date.now();
+		let removedCount = 0;
+
+		for (const l1Key in dataHolder) {
+			if (dataHolder.hasOwnProperty(l1Key)) {
+				const tenantData = dataHolder[l1Key];
+
+				for (const l2Key in tenantData) {
+					if (tenantData.hasOwnProperty(l2Key)) {
+						const entry = tenantData[l2Key];
+						if (entry.expiresAt && now > entry.expiresAt) {
+							delete tenantData[l2Key];
+							removedCount++;
+						}
+					}
+				}
+
+				if (Object.keys(tenantData).length === 0) {
+					delete dataHolder[l1Key];
+				}
+			}
+		}
+
+		if (removedCount > 0 && model.log) {
+			model.log.debug(`Idempotency cache cleanup: removed ${removedCount} expired entries`);
+		}
+	},
+
+	'get': (key) => {
+		if (!dataHolder[key.l1]) {
+			return null;
+		}
+		const entry = dataHolder[key.l1][key.l2];
+		if (!entry) {
+			return null;
+		}
+		if (entry.expiresAt && Date.now() > entry.expiresAt) {
+			delete dataHolder[key.l1][key.l2];
+			return null;
+		}
+		return entry;
+	},
+
+	'lock': (key, ttl) => {
+		if (!dataHolder[key.l1]) {
+			dataHolder[key.l1] = {};
+		}
+		const existing = dataHolder[key.l1][key.l2];
+		if (existing && existing.expiresAt && Date.now() <= existing.expiresAt) {
+			return false;
+		}
+		dataHolder[key.l1][key.l2] = {
+			'status': 'in_flight',
+			'lockedAt': Date.now(),
+			'expiresAt': Date.now() + ttl
+		};
+		return true;
+	},
+
+	'complete': (key, response, ttl) => {
+		if (!dataHolder[key.l1]) {
+			dataHolder[key.l1] = {};
+		}
+		dataHolder[key.l1][key.l2] = {
+			'status': 'completed',
+			'response': response,
+			'completedAt': Date.now(),
+			'expiresAt': Date.now() + ttl
+		};
+	},
+
+	'unlock': (key) => {
+		if (dataHolder[key.l1] && dataHolder[key.l1][key.l2]) {
+			delete dataHolder[key.l1][key.l2];
+		}
+	},
+
+	'shutdown': () => {
+		if (cleanupInterval) {
+			clearInterval(cleanupInterval);
+			cleanupInterval = null;
+		}
+		dataHolder = {};
+		if (model.log) {
+			model.log.info('Idempotency memory model shutdown complete');
+		}
+	}
+};
+
+if (typeof process !== 'undefined') {
+	const shutdownHandler = () => {
+		model.shutdown();
+	};
+	process.once('SIGTERM', shutdownHandler);
+	process.once('SIGINT', shutdownHandler);
+}
+
+module.exports = model;

package/mw/idempotency/model/mongo.js

@@ -0,0 +1,116 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const core = require('soajs.core.modules');
+const Mongo = core.mongo;
+
+const idempotency_store = 'idempotency_store';
+
+let model = {
+	'mongo': null,
+	'log': null,
+
+	'init': (log, dbConfiguration) => {
+		model.log = log;
+		if (!model.mongo) {
+			model.mongo = new Mongo(dbConfiguration);
+			model.mongo.createIndex(idempotency_store, { 'l1': 1, 'l2': 1 }, { unique: true }, (err, index) => {
+				log.debug("Idempotency index: " + index + " created with error: " + err);
+			});
+			model.mongo.createIndex(idempotency_store, { 'expiresAt': 1 }, { expireAfterSeconds: 0 }, (err, index) => {
+				log.debug("Idempotency TTL index: " + index + " created with error: " + err);
+			});
+		}
+	},
+
+	'get': async (key) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			const entry = await model.mongo.findOne(idempotency_store, condition);
+			if (!entry) {
+				return null;
+			}
+			if (entry.expiresAt && Date.now() > entry.expiresAt.getTime()) {
+				return null;
+			}
+			return entry;
+		} catch (e) {
+			model.log.error('Idempotency get:', e.message);
+			return null;
+		}
+	},
+
+	'lock': async (key, ttl) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			const existing = await model.mongo.findOne(idempotency_store, condition);
+			if (existing && existing.expiresAt && Date.now() <= existing.expiresAt.getTime()) {
+				return false;
+			}
+
+			let doc = {
+				'l1': key.l1,
+				'l2': key.l2,
+				'status': 'in_flight',
+				'lockedAt': new Date(),
+				'expiresAt': new Date(Date.now() + ttl)
+			};
+			let options = { 'upsert': true };
+			let s = { '$set': doc };
+			await model.mongo.updateOne(idempotency_store, condition, s, options);
+			return true;
+		} catch (e) {
+			if (e.code === 11000) {
+				return false;
+			}
+			model.log.error('Idempotency lock:', e.message);
+			return false;
+		}
+	},
+
+	'complete': async (key, response, ttl) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		let doc = {
+			'status': 'completed',
+			'response': response,
+			'completedAt': new Date(),
+			'expiresAt': new Date(Date.now() + ttl)
+		};
+		let s = { '$set': doc };
+		try {
+			await model.mongo.updateOne(idempotency_store, condition, s, {});
+		} catch (e) {
+			model.log.error('Idempotency complete:', e.message);
+		}
+	},
+
+	'unlock': async (key) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			await model.mongo.deleteOne(idempotency_store, condition);
+		} catch (e) {
+			model.log.error('Idempotency unlock:', e.message);
+		}
+	}
+};
+
+module.exports = model;

package/mw/mt/lib.js

@@ -13,7 +13,7 @@ let _ = require("lodash");
 
 module.exports = {
 	"mergeACLArray": function (arrayACL, cb) {
-		
+
 		//if only one return it
 		if (arrayACL.length === 1) {
 			return cb(null, arrayACL[0]);
@@ -30,30 +30,30 @@ module.exports = {
 			function () {
 				return cb(null, acl);
 			});
-		
+
 		function mergePackageAcl(oneAcl, cb) {
 			//first object
 			if (!acl.acl) {
-				acl.acl = oneAcl.acl;
+				acl.acl = _.cloneDeep(oneAcl.acl);
 				return cb();
 			}
 			async.eachOfSeries(oneAcl.acl, function (service, serviceName, serviceCall) {
 				if (!acl.acl[serviceName]) {
-					acl.acl[serviceName] = service;
+					acl.acl[serviceName] = _.cloneDeep(service);
 					return serviceCall();
 				}
 				async.eachOfSeries(service, function (version, versionValue, versionCall) {
 					if (!acl.acl[serviceName][versionValue]) {
-						acl.acl[serviceName][versionValue] = version;
+						acl.acl[serviceName][versionValue] = _.cloneDeep(version);
 						return serviceCall();
 					}
 					if (version.apisPermission === "restricted") {
 						if (acl.acl[serviceName] && acl.acl[serviceName][versionValue] && acl.acl[serviceName][versionValue].apisPermission && acl.acl[serviceName][versionValue].apisPermission === "restricted") {
 							//only restricted allowed to merge
-							acl.acl[serviceName][versionValue] = _.merge(acl.acl[serviceName][versionValue], version);
+							acl.acl[serviceName][versionValue] = _.merge(acl.acl[serviceName][versionValue], _.cloneDeep(version));
 							return versionCall();
 						} else {
-							acl.acl[serviceName][versionValue] = version;
+							acl.acl[serviceName][versionValue] = _.cloneDeep(version);
 							return versionCall();
 						}
 					} else {
@@ -62,42 +62,42 @@ module.exports = {
 							return versionCall();
 						} else {
 							//only non restricted allowed to merge
-							acl.acl[serviceName][versionValue] = _.merge(acl.acl[serviceName][versionValue], version);
+							acl.acl[serviceName][versionValue] = _.merge(acl.acl[serviceName][versionValue], _.cloneDeep(version));
 							return versionCall();
 						}
 					}
 				}, serviceCall);
 			}, cb);
 		}
-		
+
 		function mergePackageAclAllEnv(oneAcl, cb) {
 			//first object
 			if (!acl.acl_all_env) {
-				acl.acl_all_env = oneAcl.acl_all_env;
+				acl.acl_all_env = _.cloneDeep(oneAcl.acl_all_env);
 				return cb();
 			}
 			async.eachOf(oneAcl.acl_all_env, function (environment, envName, envCall) {
 				if (!acl.acl_all_env[envName]) {
-					acl.acl_all_env[envName] = environment;
+					acl.acl_all_env[envName] = _.cloneDeep(environment);
 					return envCall();
 				}
 				async.eachOf(environment, function (service, serviceName, serviceCall) {
 					if (!acl.acl_all_env[envName][serviceName]) {
-						acl.acl_all_env[envName][serviceName] = service;
+						acl.acl_all_env[envName][serviceName] = _.cloneDeep(service);
 						return serviceCall();
 					}
 					async.eachOf(service, function (version, versionValue, versionCall) {
 						if (!acl.acl_all_env[envName][serviceName][versionValue]) {
-							acl.acl_all_env[envName][serviceName][versionValue] = version;
+							acl.acl_all_env[envName][serviceName][versionValue] = _.cloneDeep(version);
 							return serviceCall();
 						}
 						if (version.apisPermission === "restricted") {
 							if (acl.acl_all_env[envName][serviceName][versionValue].apisPermission && acl.acl_all_env[envName][serviceName][versionValue].apisPermission === "restricted") {
 								//only restricted allowed to merge
-								acl.acl_all_env[envName][serviceName][versionValue] = _.merge(acl.acl_all_env[envName][serviceName][versionValue], version);
+								acl.acl_all_env[envName][serviceName][versionValue] = _.merge(acl.acl_all_env[envName][serviceName][versionValue], _.cloneDeep(version));
 								return versionCall();
 							} else {
-								acl.acl_all_env[envName][serviceName][versionValue] = version;
+								acl.acl_all_env[envName][serviceName][versionValue] = _.cloneDeep(version);
 								return versionCall();
 							}
 						} else {
@@ -106,7 +106,7 @@ module.exports = {
 								return versionCall();
 							} else {
 								//only non restricted allowed to merge
-								acl.acl_all_env[envName][serviceName][versionValue] = _.merge(acl.acl_all_env[envName][serviceName][versionValue], version);
+								acl.acl_all_env[envName][serviceName][versionValue] = _.merge(acl.acl_all_env[envName][serviceName][versionValue], _.cloneDeep(version));
 								return versionCall();
 							}
 						}
@@ -115,4 +115,4 @@ module.exports = {
 			}, cb);
 		}
 	}
-};
+};

package/mw/soajsRoutes/keyACL/index.js

@@ -31,7 +31,7 @@ module.exports = (configuration) => {
 		}
 		if (!ACL && req.soajs.controller.serviceParams.packObj.acl) {
 			req.soajs.log.debug("soajsRoute keyACL: Found Default ACL at Package level, setting default ACL configuration.");
-			ACL = req.soajs.controller.serviceParams.packObj.acl;
+			ACL = coreLibs.utils.cloneObj(req.soajs.controller.serviceParams.packObj.acl);
 		}
 		let finalACL = null;
 		if (ACL) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "soajs.controller",
   "description": "soajs multi tenant API gateway",
-  "version": "4.3.3",
+  "version": "4.3.5",
   "author": {
     "name": "soajs team",
     "email": "team@soajs.org"
@@ -27,9 +27,9 @@
     "merge": "2.1.1",
     "netmask": "2.0.2",
     "oauth2-server": "2.4.1",
-    "path-to-regexp": "8.3.0",
+    "path-to-regexp": "8.4.0",
     "soajs.core.libs": "1.2.2",
-    "soajs.core.modules": "5.2.18",
+    "soajs.core.modules": "5.2.19",
     "soajs.urac.driver": "3.2.15",
     "useragent": "2.3.0"
   },

package/server/controller.js

@@ -37,6 +37,8 @@ const keyACL_mw = require("../mw/keyACL/index");
 const gotoService_mw = require("../mw/gotoService/index");
 const mt_mw = require("../mw/mt/index");
 const traffic_mw = require("../mw/traffic/index");
+const idempotency_mw = require("../mw/idempotency/index");
+const cache_mw = require("../mw/cache/index");
 const lastSeen_mw = require("../mw/lastSeen/index");
 const ip2ban_mw = require("../mw/ip2ban/index");
 
@@ -175,11 +177,23 @@ Controller.prototype.init = function (callback) {
 					}));
 					log.info("SOAJS MT middleware initialization done.");
 
+					app.use(idempotency_mw({
+						"log": log,
+						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
+					}));
+					log.info("Idempotency middleware initialization done.");
+
 					app.use(traffic_mw({
 						"log": log,
 						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
 					}));
 
+					app.use(cache_mw({
+						"log": log,
+						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
+					}));
+					log.info("Cache middleware initialization done.");
+
 					app.use(lastSeen_mw());
 
 					app.use((req, res, next) => {

package/test/integration/UC4-multiuser-acl/readme.md

@@ -0,0 +1,11 @@
+- i have 2 users with access to the same tenant
+- under this tenant i have 2 groups as follow:
+    - owner with the following packages [GRPCH_DEFAU, GRPCH_OWNER]
+    - member with the following packages [GRPCH_DEFAU, GRPCH_AMMBR, GRPCH_EGDTL, GRPCH_SMDIA, GRPCH_SMSSG, GRPCH_VMMBR]
+- i have microservice connectonboarding with an api DELETE /group/chats/users/:userId and this api is only under package GRPCH_OWNER
+
+i have the following usecase:
+- load the provision by hitting the maintenance route /loadProvision
+- user1 with member group login first and hit the acl api /soajs/acl, then we check the response and we see that he has the right packages and api DELETE /group/chats/users/:userId is not part of finalACL which is correct
+- user2 with owner group login  and hit the acl api /soajs/acl, then we check the response and we see that he has the right packages and api DELETE /group/chats/users/:userId is  part of finalACL which is correct
+- user1 with member group hit the acl api /soajs/acl again, then we check the response and we see that he has the right packages but api DELETE /group/chats/users/:userId is now part of finalACL which is not correct

package/test/unit/index.js

@@ -34,6 +34,10 @@ describe("Starting Gateway Unit test", () => {
         require("./mw/response/index.js");
         require("./mw/soajs/index.js");
         require("./mw/traffic/index.js");
+        require("./mw/idempotency/index.js");
+        require("./mw/idempotency/memory.js");
+        require("./mw/cache/index.js");
+        require("./mw/cache/memory.js");
         require("./mw/lastSeen/index.js");
 
         require("./utilities/utils.js");