soajs.controller 4.3.4 → 4.3.5

package/mw/idempotency/index.js

@@ -0,0 +1,171 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
+const memory = require('./model/memory.js');
+const mongo = require('./model/mongo.js');
+
+let model = { memory, mongo };
+
+const UUID_V4_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+function isValidUUIDv4(str) {
+	return UUID_V4_REGEX.test(str);
+}
+
+function matchesApi(method, path, apis) {
+	if (!apis || !Array.isArray(apis)) {
+		return false;
+	}
+	const requestSignature = `${method} ${path}`;
+	for (let api of apis) {
+		if (api === requestSignature) {
+			return true;
+		}
+		const [apiMethod, apiPath] = api.split(' ');
+		if (apiMethod !== method) {
+			continue;
+		}
+		const apiParts = apiPath.split('/');
+		const pathParts = path.split('/');
+		if (apiParts.length !== pathParts.length) {
+			continue;
+		}
+		let matches = true;
+		for (let i = 0; i < apiParts.length; i++) {
+			if (apiParts[i].startsWith(':')) {
+				continue;
+			}
+			if (apiParts[i] !== pathParts[i]) {
+				matches = false;
+				break;
+			}
+		}
+		if (matches) {
+			return true;
+		}
+	}
+	return false;
+}
+
+module.exports = function (configuration) {
+	model.memory.init(configuration.log);
+	model.mongo.init(configuration.log, configuration.gatewayDB);
+
+	return (req, res, next) => {
+		const idempotencyKey = req.headers['idempotency-key'];
+
+		if (!idempotencyKey) {
+			return next();
+		}
+
+		if (!isValidUUIDv4(idempotencyKey)) {
+			req.soajs.controllerResponse({
+				'status': 400,
+				'code': 180,
+				'msg': 'Invalid Idempotency-Key format. Expected UUID v4.'
+			});
+			return;
+		}
+
+		if (!req.soajs || !req.soajs.tenant || !req.soajs.controller || !req.soajs.controller.serviceParams) {
+			return next();
+		}
+
+		const serviceName = req.soajs.controller.serviceParams.name;
+		// Strip service prefix from pathname to get API path (e.g., /console/registry -> /registry)
+		const serviceInfo = req.soajs.controller.serviceParams.serviceInfo;
+		const apiPath = "/" + serviceInfo.slice(2).join("/");
+		const method = req.method.toUpperCase();
+
+		if (method === 'GET') {
+			return next();
+		}
+
+		const idempotencyConfig = get(["soajs", "registry", "custom", "gateway", "value", "idempotency"], req);
+		if (!idempotencyConfig) {
+			return next();
+		}
+
+		const serviceConfig = idempotencyConfig.services && idempotencyConfig.services[serviceName];
+		if (!serviceConfig || !serviceConfig.enabled) {
+			return next();
+		}
+
+		if (serviceConfig.apis && !matchesApi(method, apiPath, serviceConfig.apis)) {
+			return next();
+		}
+
+		const modelType = idempotencyConfig.model || 'memory';
+		if (!model[modelType]) {
+			configuration.log.warn('Idempotency: Unknown model [' + modelType + ']. It can only be [memory || mongo]');
+			return next();
+		}
+
+		const ttl = serviceConfig.ttl || 60000;
+		const key = {
+			'l1': req.soajs.tenant.id,
+			'l2': idempotencyKey
+		};
+
+		const processIdempotency = async () => {
+			try {
+				const existing = await model[modelType].get(key);
+
+				if (existing) {
+					if (existing.status === 'in_flight') {
+						req.soajs.controllerResponse({
+							'status': 409,
+							'code': 181,
+							'msg': 'Request with this Idempotency-Key is still being processed.'
+						});
+						return;
+					}
+
+					if (existing.status === 'completed' && existing.response) {
+						res.writeHead(existing.response.statusCode, existing.response.headers);
+						res.end(existing.response.body);
+						return;
+					}
+				}
+
+				const locked = await model[modelType].lock(key, ttl);
+				if (!locked) {
+					req.soajs.controllerResponse({
+						'status': 409,
+						'code': 181,
+						'msg': 'Request with this Idempotency-Key is still being processed.'
+					});
+					return;
+				}
+
+				// Store idempotency context for gotoService to use
+				req.soajs.idempotencyContext = {
+					key: key,
+					ttl: ttl,
+					modelType: modelType,
+					model: model[modelType],
+					log: configuration.log
+				};
+
+				next();
+			} catch (err) {
+				configuration.log.error('Idempotency error:', err);
+				model[modelType].unlock(key).catch(() => {});
+				next();
+			}
+		};
+
+		processIdempotency();
+	};
+};
+
+// Export model for external use
+module.exports.model = model;

package/mw/idempotency/model/memory.js

@@ -0,0 +1,128 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+let dataHolder = {};
+let cleanupInterval = null;
+
+let model = {
+	'log': null,
+
+	'init': (log) => {
+		model.log = log;
+		dataHolder = {};
+
+		if (cleanupInterval) {
+			clearInterval(cleanupInterval);
+		}
+		const CLEANUP_INTERVAL = process.env.SOAJS_IDEMPOTENCY_CLEANUP_INTERVAL || 60000;
+		cleanupInterval = setInterval(() => {
+			model.cleanupExpiredEntries();
+		}, CLEANUP_INTERVAL);
+	},
+
+	'cleanupExpiredEntries': () => {
+		const now = Date.now();
+		let removedCount = 0;
+
+		for (const l1Key in dataHolder) {
+			if (dataHolder.hasOwnProperty(l1Key)) {
+				const tenantData = dataHolder[l1Key];
+
+				for (const l2Key in tenantData) {
+					if (tenantData.hasOwnProperty(l2Key)) {
+						const entry = tenantData[l2Key];
+						if (entry.expiresAt && now > entry.expiresAt) {
+							delete tenantData[l2Key];
+							removedCount++;
+						}
+					}
+				}
+
+				if (Object.keys(tenantData).length === 0) {
+					delete dataHolder[l1Key];
+				}
+			}
+		}
+
+		if (removedCount > 0 && model.log) {
+			model.log.debug(`Idempotency cache cleanup: removed ${removedCount} expired entries`);
+		}
+	},
+
+	'get': (key) => {
+		if (!dataHolder[key.l1]) {
+			return null;
+		}
+		const entry = dataHolder[key.l1][key.l2];
+		if (!entry) {
+			return null;
+		}
+		if (entry.expiresAt && Date.now() > entry.expiresAt) {
+			delete dataHolder[key.l1][key.l2];
+			return null;
+		}
+		return entry;
+	},
+
+	'lock': (key, ttl) => {
+		if (!dataHolder[key.l1]) {
+			dataHolder[key.l1] = {};
+		}
+		const existing = dataHolder[key.l1][key.l2];
+		if (existing && existing.expiresAt && Date.now() <= existing.expiresAt) {
+			return false;
+		}
+		dataHolder[key.l1][key.l2] = {
+			'status': 'in_flight',
+			'lockedAt': Date.now(),
+			'expiresAt': Date.now() + ttl
+		};
+		return true;
+	},
+
+	'complete': (key, response, ttl) => {
+		if (!dataHolder[key.l1]) {
+			dataHolder[key.l1] = {};
+		}
+		dataHolder[key.l1][key.l2] = {
+			'status': 'completed',
+			'response': response,
+			'completedAt': Date.now(),
+			'expiresAt': Date.now() + ttl
+		};
+	},
+
+	'unlock': (key) => {
+		if (dataHolder[key.l1] && dataHolder[key.l1][key.l2]) {
+			delete dataHolder[key.l1][key.l2];
+		}
+	},
+
+	'shutdown': () => {
+		if (cleanupInterval) {
+			clearInterval(cleanupInterval);
+			cleanupInterval = null;
+		}
+		dataHolder = {};
+		if (model.log) {
+			model.log.info('Idempotency memory model shutdown complete');
+		}
+	}
+};
+
+if (typeof process !== 'undefined') {
+	const shutdownHandler = () => {
+		model.shutdown();
+	};
+	process.once('SIGTERM', shutdownHandler);
+	process.once('SIGINT', shutdownHandler);
+}
+
+module.exports = model;

package/mw/idempotency/model/mongo.js

@@ -0,0 +1,116 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const core = require('soajs.core.modules');
+const Mongo = core.mongo;
+
+const idempotency_store = 'idempotency_store';
+
+let model = {
+	'mongo': null,
+	'log': null,
+
+	'init': (log, dbConfiguration) => {
+		model.log = log;
+		if (!model.mongo) {
+			model.mongo = new Mongo(dbConfiguration);
+			model.mongo.createIndex(idempotency_store, { 'l1': 1, 'l2': 1 }, { unique: true }, (err, index) => {
+				log.debug("Idempotency index: " + index + " created with error: " + err);
+			});
+			model.mongo.createIndex(idempotency_store, { 'expiresAt': 1 }, { expireAfterSeconds: 0 }, (err, index) => {
+				log.debug("Idempotency TTL index: " + index + " created with error: " + err);
+			});
+		}
+	},
+
+	'get': async (key) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			const entry = await model.mongo.findOne(idempotency_store, condition);
+			if (!entry) {
+				return null;
+			}
+			if (entry.expiresAt && Date.now() > entry.expiresAt.getTime()) {
+				return null;
+			}
+			return entry;
+		} catch (e) {
+			model.log.error('Idempotency get:', e.message);
+			return null;
+		}
+	},
+
+	'lock': async (key, ttl) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			const existing = await model.mongo.findOne(idempotency_store, condition);
+			if (existing && existing.expiresAt && Date.now() <= existing.expiresAt.getTime()) {
+				return false;
+			}
+
+			let doc = {
+				'l1': key.l1,
+				'l2': key.l2,
+				'status': 'in_flight',
+				'lockedAt': new Date(),
+				'expiresAt': new Date(Date.now() + ttl)
+			};
+			let options = { 'upsert': true };
+			let s = { '$set': doc };
+			await model.mongo.updateOne(idempotency_store, condition, s, options);
+			return true;
+		} catch (e) {
+			if (e.code === 11000) {
+				return false;
+			}
+			model.log.error('Idempotency lock:', e.message);
+			return false;
+		}
+	},
+
+	'complete': async (key, response, ttl) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		let doc = {
+			'status': 'completed',
+			'response': response,
+			'completedAt': new Date(),
+			'expiresAt': new Date(Date.now() + ttl)
+		};
+		let s = { '$set': doc };
+		try {
+			await model.mongo.updateOne(idempotency_store, condition, s, {});
+		} catch (e) {
+			model.log.error('Idempotency complete:', e.message);
+		}
+	},
+
+	'unlock': async (key) => {
+		let condition = {
+			'l1': key.l1,
+			'l2': key.l2
+		};
+		try {
+			await model.mongo.deleteOne(idempotency_store, condition);
+		} catch (e) {
+			model.log.error('Idempotency unlock:', e.message);
+		}
+	}
+};
+
+module.exports = model;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "soajs.controller",
   "description": "soajs multi tenant API gateway",
-  "version": "4.3.4",
+  "version": "4.3.5",
   "author": {
     "name": "soajs team",
     "email": "team@soajs.org"

package/server/controller.js

@@ -37,6 +37,8 @@ const keyACL_mw = require("../mw/keyACL/index");
 const gotoService_mw = require("../mw/gotoService/index");
 const mt_mw = require("../mw/mt/index");
 const traffic_mw = require("../mw/traffic/index");
+const idempotency_mw = require("../mw/idempotency/index");
+const cache_mw = require("../mw/cache/index");
 const lastSeen_mw = require("../mw/lastSeen/index");
 const ip2ban_mw = require("../mw/ip2ban/index");
 
@@ -175,11 +177,23 @@ Controller.prototype.init = function (callback) {
 					}));
 					log.info("SOAJS MT middleware initialization done.");
 
+					app.use(idempotency_mw({
+						"log": log,
+						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
+					}));
+					log.info("Idempotency middleware initialization done.");
+
 					app.use(traffic_mw({
 						"log": log,
 						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
 					}));
 
+					app.use(cache_mw({
+						"log": log,
+						"gatewayDB": registry.coreDB.gateway || registry.coreDB.provision
+					}));
+					log.info("Cache middleware initialization done.");
+
 					app.use(lastSeen_mw());
 
 					app.use((req, res, next) => {

package/test/unit/index.js

@@ -34,6 +34,10 @@ describe("Starting Gateway Unit test", () => {
         require("./mw/response/index.js");
         require("./mw/soajs/index.js");
         require("./mw/traffic/index.js");
+        require("./mw/idempotency/index.js");
+        require("./mw/idempotency/memory.js");
+        require("./mw/cache/index.js");
+        require("./mw/cache/memory.js");
         require("./mw/lastSeen/index.js");
 
         require("./utilities/utils.js");