soajs.controller 4.3.18 → 4.3.19

package/.github/workflows/ci.yml

@@ -17,7 +17,10 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:8
+        # TODO: back to mongo:8 once it starts on the GitHub runners again. Since 2026-10-19 mongod 8
+        #       exits on start with "Linux kernel versions 6.19 and newer has a known incompatibility
+        #       with this version of MongoDB" (https://jira.mongodb.org/browse/SERVER-121912).
+        image: mongo:7.0
         ports:
           - 27017:27017
 

package/.github/workflows/release.yml

@@ -17,7 +17,10 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:8
+        # TODO: back to mongo:8 once it starts on the GitHub runners again. Since 2026-10-19 mongod 8
+        #       exits on start with "Linux kernel versions 6.19 and newer has a known incompatibility
+        #       with this version of MongoDB" (https://jira.mongodb.org/browse/SERVER-121912).
+        image: mongo:7.0
         ports:
           - 27017:27017
 

package/docs/middleware/idempotency.md

@@ -48,7 +48,8 @@ The Idempotency middleware prevents duplicate write operations by tracking reque
            │
            ├─── Found (in_flight) ──▶ 409 Conflict (code: 181)
            │
-           └─── Not found ──▶ Lock key → Execute → Store response
+           └─── Not found ──▶ Lock key → Execute → Store response (status < 500)
+                                                 or release the key
 ```
 
 ## Configuration
@@ -140,12 +141,39 @@ Content-Type: application/json
 {"caller": "user1", "callee": "user2"}
 ```
 
+## How a Locked Key Ends
+
+The key is locked before the request goes on to throttling, caching and the proxy. How the
+request ends decides what happens to it:
+
+| The request ends with | The key | A retry with the same key |
+|---|---|---|
+| An answer of the service below 500 (2xx, 3xx, 4xx) | Completed, the answer is stored for the TTL | Gets the stored answer, the service is not called |
+| An answer of the service of 500 or more | Released | Reaches the service again |
+| A proxy error (service down, connection closed) | Released | Reaches the service again |
+| An answer of the gateway before the service: throttling (429), an error, a refusal | Released | Runs again |
+| The client disconnecting before the request reaches the service | Released | Runs again |
+| The client disconnecting while the service is answering | Follows the service answer above | |
+
+Only an answer the service actually produced is replayed. A 4xx is the service's answer to that
+request and replaying it is correct. A 5xx, a throttling refusal or any other answer of the gateway
+says nothing about the operation, so the key is released and the client's retry runs instead of
+getting the same refusal for the whole TTL.
+
+While the request is running, a duplicate with the same key gets 409, code 181.
+
+A service that can fail with a 5xx after part of the work is done must protect that work itself,
+the retry reaches it again.
+
+The release only removes a key still in flight and locked by that request: never a completed
+one, and never a lock taken by a retry after the TTL of the first ran out.
+
 ## Response Scenarios
 
 ### New Request (First Time)
 
 ```
-Request processed normally, response cached
+Request processed normally, a service answer below 500 is stored
 ```
 
 ### Replay (Completed)
@@ -235,6 +263,7 @@ Keys are isolated per tenant to prevent cross-tenant collisions.
 - Only applies to non-GET requests (POST, PUT, DELETE, PATCH)
 - GET requests always pass through (idempotent by nature)
 - Keys expire after TTL (default 60 seconds)
+- A key is never left in flight by a request that does not reach the service, see [How a Locked Key Ends](#how-a-locked-key-ends)
 - Clients should generate a new UUID for each unique operation
 - Same UUID = same operation (for retries)
 - MongoDB backend required for horizontal scaling

package/mw/gotoService/redirectToService.js

@@ -11,6 +11,7 @@
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 
 const { httpRequestLight, proxyRequestMonitor } = require("../../lib/request.js");
+const idempotency_mw = require("../idempotency/index.js");
 
 // Monitor buffer for batch uploads
 let monitorBuffer = [];
@@ -395,6 +396,11 @@ module.exports = (configuration) => {
 				};
 			}
 
+			//NOTE: from here the outcome of the service decides the key, the release on finish in the
+			//		idempotency middleware stands down
+			if (idempotencyContext) {
+				idempotencyContext.proxied = true;
+			}
 			proxyRequestMonitor(req, res, requestOptions, extraOptions)
 				.then(() => {
 					// Save to cache if applicable
@@ -408,14 +414,20 @@ module.exports = (configuration) => {
 						cacheContext.model.set(cacheContext.key, cacheData, cacheContext.ttl);
 					}
 					// Complete idempotency if applicable
+					//NOTE: only an answer of the service is replayed, a 5xx means it could not answer, release
+					//		the key so the retry reaches the service again
 					if (idempotencyContext) {
-						let idempotencyBody = Buffer.concat(idempotencyResponseChunks).toString('utf8');
-						let idempotencyData = {
-							statusCode: idempotencyResponseStatusCode,
-							headers: idempotencyResponseHeaders,
-							body: idempotencyBody
-						};
-						idempotencyContext.model.complete(idempotencyContext.key, idempotencyData, idempotencyContext.ttl);
+						if (idempotencyResponseStatusCode < 500) {
+							let idempotencyBody = Buffer.concat(idempotencyResponseChunks).toString('utf8');
+							let idempotencyData = {
+								statusCode: idempotencyResponseStatusCode,
+								headers: idempotencyResponseHeaders,
+								body: idempotencyBody
+							};
+							idempotencyContext.model.complete(idempotencyContext.key, idempotencyData, idempotencyContext.ttl);
+						} else {
+							idempotency_mw.unlock(idempotencyContext.model, idempotencyContext.key, idempotencyContext.lockId, req.soajs.log);
+						}
 					}
 					if (monitor && !monitor_service_blacklist && monitor.req_response) {
 						monitoObj.time.res_end = new Date().getTime();
@@ -430,7 +442,7 @@ module.exports = (configuration) => {
 					}
 					// Unlock idempotency on error
 					if (idempotencyContext) {
-						idempotencyContext.model.unlock(idempotencyContext.key);
+						idempotency_mw.unlock(idempotencyContext.model, idempotencyContext.key, idempotencyContext.lockId, req.soajs.log);
 					}
 					if (!req.soajs.controller.monitorEndingReq) {
 						req.soajs.controllerResponse(core.error.getError(135));

package/mw/idempotency/index.js

@@ -8,6 +8,7 @@
  * found in the LICENSE file at the root of this repository
  */
 
+const crypto = require('crypto');
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 const memory = require('./model/memory.js');
 const mongo = require('./model/mongo.js');
@@ -56,6 +57,21 @@ function matchesApi(method, path, apis) {
 	return false;
 }
 
+/**
+ * Release a key this request locked, only while it is still in flight and still ours
+ * @param modelObj memory or mongo
+ * @param key
+ * @param lockId
+ * @param log
+ */
+function unlock(modelObj, key, lockId, log) {
+	Promise.resolve()
+		.then(() => modelObj.unlock(key, lockId))
+		.catch((err) => {
+			log.error('Idempotency unlock:', err.message);
+		});
+}
+
 module.exports = function (configuration) {
 	model.memory.init(configuration.log);
 	model.mongo.init(configuration.log, configuration.gatewayDB);
@@ -131,6 +147,7 @@ module.exports = function (configuration) {
 			'l2': idempotencyKey
 		};
 
+		let lockId = null;
 		const processIdempotency = async () => {
 			try {
 				const existing = await model[modelType].get(key);
@@ -152,7 +169,8 @@ module.exports = function (configuration) {
 					}
 				}
 
-				const locked = await model[modelType].lock(key, ttl);
+				lockId = crypto.randomUUID();
+				const locked = await model[modelType].lock(key, ttl, lockId);
 				if (!locked) {
 					req.soajs.controllerResponse({
 						'status': 409,
@@ -163,18 +181,37 @@ module.exports = function (configuration) {
 				}
 
 				// Store idempotency context for gotoService to use
-				req.soajs.idempotencyContext = {
+				const context = {
 					key: key,
 					ttl: ttl,
+					lockId: lockId,
 					modelType: modelType,
 					model: model[modelType],
-					log: configuration.log
+					log: configuration.log,
+					proxied: false
 				};
+				req.soajs.idempotencyContext = context;
+
+				//NOTE: a request answered before it reaches the service, throttled, refused or dropped by
+				//		the client, must not keep the key in flight until the ttl, release it once the response
+				//		is over. A proxied request is completed or released by gotoService, never here.
+				let released = false;
+				const release = () => {
+					if (released || context.proxied) {
+						return;
+					}
+					released = true;
+					unlock(model[modelType], key, lockId, configuration.log);
+				};
+				res.on('finish', release);
+				res.on('close', release);
 
 				next();
 			} catch (err) {
 				configuration.log.error('Idempotency error:', err);
-				model[modelType].unlock(key).catch(() => {});
+				if (lockId) {
+					unlock(model[modelType], key, lockId, configuration.log);
+				}
 				next();
 			}
 		};
@@ -185,3 +222,4 @@ module.exports = function (configuration) {
 
 // Export model for external use
 module.exports.model = model;
+module.exports.unlock = unlock;

package/mw/idempotency/model/memory.js

@@ -75,7 +75,7 @@ let model = {
 		return entry;
 	},
 
-	'lock': (key, ttl) => {
+	'lock': (key, ttl, lockId) => {
 		if (!dataHolder[key.l1]) {
 			dataHolder[key.l1] = {};
 		}
@@ -85,6 +85,7 @@ let model = {
 		}
 		dataHolder[key.l1][key.l2] = {
 			'status': 'in_flight',
+			'lockId': lockId || null,
 			'lockedAt': Date.now(),
 			'expiresAt': Date.now() + ttl
 		};
@@ -103,8 +104,17 @@ let model = {
 		};
 	},
 
-	'unlock': (key) => {
+	//NOTE: only an entry still in flight is removed, never a completed one, and with a lockId only
+	//		the lock of that request, not one taken by a retry after the ttl
+	'unlock': (key, lockId) => {
 		if (dataHolder[key.l1] && dataHolder[key.l1][key.l2]) {
+			const entry = dataHolder[key.l1][key.l2];
+			if (entry.status !== 'in_flight') {
+				return;
+			}
+			if (lockId && entry.lockId !== lockId) {
+				return;
+			}
 			delete dataHolder[key.l1][key.l2];
 		}
 	},

package/mw/idempotency/model/mongo.js

@@ -58,7 +58,7 @@ let model = {
 		}
 	},
 
-	'lock': async (key, ttl) => {
+	'lock': async (key, ttl, lockId) => {
 		let condition = {
 			'l1': key.l1,
 			'l2': key.l2
@@ -73,6 +73,7 @@ let model = {
 				'l1': key.l1,
 				'l2': key.l2,
 				'status': 'in_flight',
+				'lockId': lockId || null,
 				'lockedAt': new Date(),
 				'expiresAt': new Date(Date.now() + ttl)
 			};
@@ -108,11 +109,17 @@ let model = {
 		}
 	},
 
-	'unlock': async (key) => {
+	//NOTE: only a record still in flight is removed, never a completed one, and with a lockId only
+	//		the lock of that request, not one taken by a retry after the ttl
+	'unlock': async (key, lockId) => {
 		let condition = {
 			'l1': key.l1,
-			'l2': key.l2
+			'l2': key.l2,
+			'status': 'in_flight'
 		};
+		if (lockId) {
+			condition.lockId = lockId;
+		}
 		try {
 			await model.mongo.deleteOne(idempotency_store, condition);
 		} catch (e) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "soajs.controller",
   "description": "soajs multi tenant API gateway",
-  "version": "4.3.18",
+  "version": "4.3.19",
   "author": {
     "name": "soajs team",
     "email": "team@soajs.org"

package/test/data/provision/customRegistry/gateway.js

@@ -0,0 +1,32 @@
+'use strict';
+let doc = {
+	"_id": "5d7f8e43edf21900fed08cf7",
+	"name": "gateway",
+	"locked": false,
+	"plugged": true,
+	"shared": false,
+	"value": {
+		"idempotency": {
+			"model": "mongo",
+			"services": {
+				"idempotent": {
+					"enabled": true,
+					"ttl": 60000,
+					"apis": ["POST /ok", "POST /fail", "POST /bad", "POST /slow", "POST /throttled"]
+				}
+			}
+		},
+		"traffic": {
+			"throttling": {
+				"idempotent": {
+					"publicAPIStrategy": "idempotent_tight",
+					"apis": ["/throttled"]
+				}
+			}
+		}
+	},
+	"created": "DEV",
+	"author": "owner"
+};
+
+module.exports = doc;

package/test/data/provision/environment/dev.js

@@ -115,6 +115,15 @@ let env = {
 	            "accessTokenLifetime": 7200,
 	            "refreshTokenLifetime": 1209600
             },
+            "throttling": {
+                "idempotent_tight": {
+                    "type": 1,
+                    "window": 60000,
+                    "limit": 1,
+                    "retries": 0,
+                    "delay": 100
+                }
+            },
             "ports": {
 	            "controller": 4000,
 	            "maintenanceInc": 1001,

package/test/data/provision/hosts/idempotent.js

@@ -0,0 +1,12 @@
+'use strict';
+
+let host = {
+    _id: "5da078e22357684b57effcf3",
+    env: "dev",
+    ip: "127.0.0.1",
+    name: "idempotent",
+    version: "1",
+    port: 4014
+};
+
+module.exports = host;

package/test/data/provision/marketplace/idempotent.js

@@ -0,0 +1,37 @@
+'use strict';
+
+let service = {
+	_id: "5da078c92357684b57effcf2",
+	"type": "service",
+	name: "idempotent",
+	configuration: {
+		group: "SOAJS test",
+		port: 4014,
+		requestTimeout: 30,
+		requestTimeoutRenewal: 5
+	},
+	versions: [{
+		"version": "1",
+		maintenance: {
+			"port": {
+				"type": "maintenance"
+			},
+			"readiness": "/heartbeat"
+		},
+		oauth: true,
+		extKeyRequired: true,
+		urac: true,
+		urac_Profile: false,
+		urac_ACL: false,
+		provision_ACL: false,
+		apis: [
+			{ l: "Answers 200", v: "/ok", m: "post", group: "Integration test" },
+			{ l: "Answers 503 the first time", v: "/fail", m: "post", group: "Integration test" },
+			{ l: "Answers 400", v: "/bad", m: "post", group: "Integration test" },
+			{ l: "Answers 200 after a delay", v: "/slow", m: "post", group: "Integration test" },
+			{ l: "Answers 200, throttled", v: "/throttled", m: "post", group: "Integration test" }
+		]
+	}
+	]
+};
+module.exports = service;

package/test/data/provision/products/dsbrd.js

@@ -9,6 +9,23 @@ let prod = {
 	"scope": {
 		"acl": {
 			"dev": {
+				"idempotent": {
+					"1": {
+						"access": true,
+						"post": [
+							{
+								"group": "Idempotent group",
+								"apis": {
+									"/ok": {},
+									"/fail": {},
+									"/bad": {},
+									"/slow": {},
+									"/throttled": {}
+								}
+							}
+						]
+					}
+				},
 				"httpmethods": {
 					"1": {
 						"access": true,
@@ -200,6 +217,14 @@ let prod = {
 			"locked": true,
 			"acl": {
 				"dev": {
+					"idempotent": [
+						{
+							"version": "1",
+							"post": [
+								"Idempotent group"
+							]
+						}
+					],
 					"httpmethods": [
 						{
 							"version": "1",

package/test/integration/UC6-idempotency/index.js

@@ -0,0 +1,197 @@
+"use strict";
+
+const assert = require('assert');
+const crypto = require('crypto');
+const express = require('express');
+const request = require('request');
+const helper = require("../../helper.js");
+let Controller = helper.requireModule('./server/controller');
+let localConfig = helper.requireModule('./config.js');
+
+//NOTE: the gateway custom registry turns idempotency on with the mongo model for every route of the
+//		idempotent service, and throttles /throttled to one call per minute
+const ownerToken = "cfb209a91b23896820f510aadbf1f4284b512123";
+
+function startService(calls, callback) {
+	const sApp = express();
+	const mApp = express();
+	let failed = false;
+	let count = (route) => {
+		calls[route] = (calls[route] || 0) + 1;
+	};
+	sApp.post('/ok', (req, res) => {
+		count("ok");
+		res.json({ "result": true, "data": { "call": calls.ok } });
+	});
+	sApp.post('/fail', (req, res) => {
+		count("fail");
+		if (!failed) {
+			failed = true;
+			return res.status(503).json({ "result": false, "errors": { "codes": [503], "details": [{ "code": 503, "message": "unavailable" }] } });
+		}
+		res.json({ "result": true, "data": { "call": calls.fail } });
+	});
+	sApp.post('/bad', (req, res) => {
+		count("bad");
+		res.status(400).json({ "result": false, "errors": { "codes": [400], "details": [{ "code": 400, "message": "bad input" }] } });
+	});
+	sApp.post('/slow', (req, res) => {
+		count("slow");
+		setTimeout(() => {
+			res.json({ "result": true, "data": { "call": calls.slow } });
+		}, 1500);
+	});
+	sApp.post('/throttled', (req, res) => {
+		count("throttled");
+		res.json({ "result": true, "data": { "call": calls.throttled } });
+	});
+	mApp.get('/heartbeat', (req, res) => {
+		res.json({ "result": true, "ts": Date.now(), "service": { "service": "idempotent", "type": "rest", "route": "/heartbeat" } });
+	});
+	let sAppServer = sApp.listen(4014, () => {
+		let mAppServer = mApp.listen(5015, () => {
+			return callback({ "sAppServer": sAppServer, "mAppServer": mAppServer });
+		});
+	});
+}
+
+describe("Integration for Usecase 6 - idempotency", function () {
+	let extKey = "3d90163cf9d6b3076ad26aa5ed58556348069258e5c6c941ee0f18448b570ad1c5c790e2d2a1989680c55f4904e2005ff5f8e71606e4aa641e67882f4210ebbc5460ff305dcb36e6ec2a2299cf0448ef60b9e38f41950ec251c1cf41f05f3ce9";
+
+	let c = new Controller({
+		"serviceName": localConfig.serviceName,
+		"serviceVersion": localConfig.serviceVersion,
+		"serviceGroup": localConfig.serviceGroup,
+		"servicePort": localConfig.servicePort,
+		"maintenance": localConfig.maintenance
+	});
+	let registry = null;
+	let log = null;
+	let service = null;
+	let server = null;
+	let serverMaintenance = null;
+	let mockServers = null;
+	let calls = {};
+
+	function call(route, key, cb) {
+		request.post({
+			"uri": 'http://127.0.0.1:' + localConfig.servicePort + '/idempotent' + route,
+			"json": true,
+			"headers": {
+				"key": extKey,
+				"idempotency-key": key,
+				"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36"
+			},
+			"qs": { "access_token": ownerToken }
+		}, (error, response, body) => {
+			assert.ifError(error);
+			return cb(response.statusCode, body);
+		});
+	}
+
+	before((done) => {
+		//NOTE: the unit tests run in the same process and connect the idempotency mongo model to a dummy
+		//		database, the model connects only once, drop it so the gateway connects it to the real one
+		helper.requireModule('./mw/idempotency/model/mongo').mongo = null;
+		startService(calls, (servers) => {
+			mockServers = servers;
+			c.init((_registry, _log, _service, _server, _serverMaintenance) => {
+				registry = _registry;
+				log = _log;
+				service = _service;
+				server = _server;
+				serverMaintenance = _serverMaintenance;
+				c.start(registry, log, service, server, serverMaintenance, () => {
+					setTimeout(function () {
+						done();
+					}, 5000);
+				});
+			});
+		});
+	});
+
+	it("a service answer is completed and replayed", function (done) {
+		let key = crypto.randomUUID();
+		call("/ok", key, (status, body) => {
+			assert.strictEqual(status, 200, JSON.stringify(body));
+			assert.strictEqual(calls.ok, 1);
+			call("/ok", key, (status, body) => {
+				assert.strictEqual(status, 200);
+				assert.strictEqual(body.data.call, 1);
+				assert.strictEqual(calls.ok, 1);
+				done();
+			});
+		});
+	});
+
+	it("a 4xx of the service is completed and replayed", function (done) {
+		let key = crypto.randomUUID();
+		call("/bad", key, (status) => {
+			assert.strictEqual(status, 400);
+			call("/bad", key, (status) => {
+				assert.strictEqual(status, 400);
+				assert.strictEqual(calls.bad, 1);
+				done();
+			});
+		});
+	});
+
+	it("a 5xx of the service releases the key, the retry reaches the service", function (done) {
+		let key = crypto.randomUUID();
+		call("/fail", key, (status) => {
+			assert.strictEqual(status, 503);
+			call("/fail", key, (status, body) => {
+				assert.strictEqual(status, 200, JSON.stringify(body));
+				assert.strictEqual(calls.fail, 2);
+				call("/fail", key, (status, body) => {
+					assert.strictEqual(status, 200);
+					assert.strictEqual(body.data.call, 2);
+					assert.strictEqual(calls.fail, 2);
+					done();
+				});
+			});
+		});
+	});
+
+	it("a duplicate while the first is running gets 409/181", function (done) {
+		let key = crypto.randomUUID();
+		let first = null;
+		call("/slow", key, (status) => {
+			first = status;
+		});
+		setTimeout(() => {
+			call("/slow", key, (status, body) => {
+				assert.strictEqual(status, 409);
+				assert.deepStrictEqual(body.errors.codes, [181]);
+				setTimeout(() => {
+					assert.strictEqual(first, 200);
+					assert.strictEqual(calls.slow, 1);
+					done();
+				}, 2000);
+			});
+		}, 300);
+	});
+
+	it("a throttled request releases the key, the retry is not refused with 409", function (done) {
+		call("/throttled", crypto.randomUUID(), (status, body) => {
+			assert.strictEqual(status, 200, JSON.stringify(body));
+			let key = crypto.randomUUID();
+			call("/throttled", key, (status) => {
+				assert.strictEqual(status, 429);
+				call("/throttled", key, (status, body) => {
+					assert.strictEqual(status, 429, JSON.stringify(body));
+					assert.strictEqual(calls.throttled, 1);
+					done();
+				});
+			});
+		});
+	});
+
+	after((done) => {
+		mockServers.sAppServer.close();
+		mockServers.mAppServer.close();
+		c.stop(registry, log, service, server, serverMaintenance, () => {
+			done();
+		});
+	});
+});

package/test/integration/index.js

@@ -40,6 +40,10 @@ describe("starting integration tests", () => {
 		// to test the tenant access resolver
 		require("./UC5-tenant-access/index.js");
 		
+		
+		// to test idempotency with the mongo model
+		require("./UC6-idempotency/index.js");
+		
 		done();
 	});
 });

package/test/unit/index.js

@@ -39,6 +39,7 @@ describe("Starting Gateway Unit test", () => {
         require("./mw/traffic/index.js");
         require("./mw/idempotency/index.js");
         require("./mw/idempotency/memory.js");
+        require("./mw/idempotency/mongo.js");
         require("./mw/cache/index.js");
         require("./mw/cache/memory.js");
         require("./mw/lastSeen/index.js");

package/test/unit/mw/idempotency/index.js

@@ -590,4 +590,126 @@ describe("Unit test for: mw - idempotency", () => {
 		});
 	});
 
+	describe("release of the key when the request ends before the service", () => {
+		const EventEmitter = require('events');
+
+		function buildReq(idempotencyKey, answers) {
+			return {
+				headers: { 'idempotency-key': idempotencyKey },
+				method: 'POST',
+				soajs: {
+					tenant: { id: 'tenant-release' },
+					controller: { serviceParams: { name: 'test', serviceInfo: ['', 'test', 'test'] } },
+					registry: {
+						custom: {
+							gateway: {
+								value: {
+									idempotency: {
+										model: 'memory',
+										services: {
+											test: { enabled: true, ttl: 60000, apis: ['POST /test'] }
+										}
+									}
+								}
+							}
+						}
+					},
+					controllerResponse: (data) => {
+						answers.push(data);
+					}
+				}
+			};
+		}
+
+		function buildRes() {
+			let res = new EventEmitter();
+			res.replayed = null;
+			res.writeHead = (statusCode) => {
+				res.replayed = statusCode;
+				return res;
+			};
+			res.end = () => {
+			};
+			return res;
+		}
+
+		// Runs the middleware and resolves with "next" or the answer it gave
+		function run(functionMw, idempotencyKey) {
+			return new Promise((resolve) => {
+				let answers = [];
+				let req = buildReq(idempotencyKey, answers);
+				let res = buildRes();
+				functionMw(req, res, () => {
+					resolve({ "next": true, req, res, answers });
+				});
+				setTimeout(() => {
+					resolve({ "next": false, req, res, answers });
+				}, 20);
+			});
+		}
+
+		let functionMw = null;
+		before(() => {
+			functionMw = mw(mockConfig);
+		});
+
+		it("an answer before the service, a 429 for example, releases the key", async () => {
+			let key = '550e8400-e29b-41d4-a716-446655440101';
+			let first = await run(functionMw, key);
+			assert.strictEqual(first.next, true);
+			first.res.emit('finish');
+			await new Promise((r) => setTimeout(r, 5));
+			let retry = await run(functionMw, key);
+			assert.strictEqual(retry.next, true, JSON.stringify(retry.answers));
+		});
+
+		it("a client disconnect before the service releases the key", async () => {
+			let key = '550e8400-e29b-41d4-a716-446655440102';
+			let first = await run(functionMw, key);
+			assert.strictEqual(first.next, true);
+			first.res.emit('close');
+			await new Promise((r) => setTimeout(r, 5));
+			let retry = await run(functionMw, key);
+			assert.strictEqual(retry.next, true, JSON.stringify(retry.answers));
+		});
+
+		it("a proxied request is left to gotoService, a duplicate still gets 409/181", async () => {
+			let key = '550e8400-e29b-41d4-a716-446655440103';
+			let first = await run(functionMw, key);
+			assert.strictEqual(first.next, true);
+			first.req.soajs.idempotencyContext.proxied = true;
+			first.res.emit('finish');
+			await new Promise((r) => setTimeout(r, 5));
+			let duplicate = await run(functionMw, key);
+			assert.strictEqual(duplicate.next, false);
+			assert.strictEqual(duplicate.answers[0].status, 409);
+			assert.strictEqual(duplicate.answers[0].code, 181);
+		});
+
+		it("a completed request is replayed, finish does not remove it", async () => {
+			let key = '550e8400-e29b-41d4-a716-446655440104';
+			let first = await run(functionMw, key);
+			let context = first.req.soajs.idempotencyContext;
+			context.proxied = true;
+			context.model.complete(context.key, { statusCode: 201, headers: {}, body: '{}' }, context.ttl);
+			first.res.emit('finish');
+			first.res.emit('close');
+			await new Promise((r) => setTimeout(r, 5));
+			let retry = await run(functionMw, key);
+			assert.strictEqual(retry.next, false);
+			assert.strictEqual(retry.res.replayed, 201);
+		});
+
+		it("the release of an expired lock does not remove the lock of a later retry", async () => {
+			let key = '550e8400-e29b-41d4-a716-446655440105';
+			let first = await run(functionMw, key);
+			let context = first.req.soajs.idempotencyContext;
+			// the ttl of the first lock ran out and a retry locked the key again
+			context.model.get(context.key).expiresAt = Date.now() - 1;
+			assert.strictEqual(context.model.lock(context.key, 60000, 'lock-of-the-retry'), true);
+			first.res.emit('finish');
+			await new Promise((r) => setTimeout(r, 5));
+			assert.strictEqual(context.model.get(context.key).lockId, 'lock-of-the-retry');
+		});
+	});
 });

package/test/unit/mw/idempotency/memory.js

@@ -16,10 +16,12 @@ describe("Unit test for: mw - idempotency/model/memory", () => {
 	});
 
 	afterEach((done) => {
-		// Clean up between tests
-		model.unlock({ l1: 'tenant1', l2: 'key1' });
-		model.unlock({ l1: 'tenant1', l2: 'key2' });
-		model.unlock({ l1: 'tenant2', l2: 'key1' });
+		// Clean up between tests, unlock leaves completed entries in place so reset the store
+		model.init({
+			debug: () => {},
+			info: () => {},
+			error: () => {}
+		});
 		done();
 	});
 
@@ -133,6 +135,36 @@ describe("Unit test for: mw - idempotency/model/memory", () => {
 		});
 	});
 
+	describe("unlock only an own lock in flight", () => {
+		it("should keep a completed entry", (done) => {
+			let key = { l1: 'tenant1', l2: 'key1' };
+			model.lock(key, 60000, 'lock-a');
+			model.complete(key, { statusCode: 200 }, 60000);
+			model.unlock(key, 'lock-a');
+			let result = model.get(key);
+			assert.strictEqual(result.status, 'completed');
+			done();
+		});
+
+		it("should keep the lock of another request", (done) => {
+			let key = { l1: 'tenant1', l2: 'key1' };
+			model.lock(key, 60000, 'lock-b');
+			model.unlock(key, 'lock-a');
+			let result = model.get(key);
+			assert.strictEqual(result.status, 'in_flight');
+			assert.strictEqual(result.lockId, 'lock-b');
+			done();
+		});
+
+		it("should remove its own lock", (done) => {
+			let key = { l1: 'tenant1', l2: 'key1' };
+			model.lock(key, 60000, 'lock-a');
+			model.unlock(key, 'lock-a');
+			assert.strictEqual(model.get(key), null);
+			done();
+		});
+	});
+
 	describe("cleanupExpiredEntries", () => {
 		it("should remove expired entries", (done) => {
 			let key1 = { l1: 'tenant1', l2: 'key1' };

package/test/unit/mw/idempotency/mongo.js

@@ -0,0 +1,62 @@
+"use strict";
+
+const helper = require("../../../helper.js");
+const model = helper.requireModule('./mw/idempotency/model/mongo');
+const assert = require('assert');
+
+describe("Unit test for: mw - idempotency/model/mongo", () => {
+
+	let calls = null;
+	let originalMongo = null;
+
+	before(() => {
+		originalMongo = model.mongo;
+		model.log = {
+			debug: () => {},
+			info: () => {},
+			error: () => {}
+		};
+		model.mongo = {
+			findOne: async (col, condition) => {
+				calls.push({ "op": "findOne", "condition": condition });
+				return null;
+			},
+			updateOne: async (col, condition, s, options) => {
+				calls.push({ "op": "updateOne", "condition": condition, "s": s, "options": options });
+			},
+			deleteOne: async (col, condition) => {
+				calls.push({ "op": "deleteOne", "condition": condition });
+			}
+		};
+	});
+	after(() => {
+		model.mongo = originalMongo;
+	});
+	beforeEach(() => {
+		calls = [];
+	});
+
+	it("lock stores the lockId", async () => {
+		let locked = await model.lock({ l1: 'tenant1', l2: 'key1' }, 60000, 'lock-a');
+		assert.strictEqual(locked, true);
+		let update = calls.find((c) => c.op === "updateOne");
+		assert.strictEqual(update.s.$set.status, 'in_flight');
+		assert.strictEqual(update.s.$set.lockId, 'lock-a');
+	});
+
+	it("unlock removes only an own record in flight", async () => {
+		await model.unlock({ l1: 'tenant1', l2: 'key1' }, 'lock-a');
+		assert.deepStrictEqual(calls, [{
+			"op": "deleteOne",
+			"condition": { l1: 'tenant1', l2: 'key1', status: 'in_flight', lockId: 'lock-a' }
+		}]);
+	});
+
+	it("unlock without a lockId still never removes a completed record", async () => {
+		await model.unlock({ l1: 'tenant1', l2: 'key1' });
+		assert.deepStrictEqual(calls, [{
+			"op": "deleteOne",
+			"condition": { l1: 'tenant1', l2: 'key1', status: 'in_flight' }
+		}]);
+	});
+});