soajs.controller 4.3.24 → 4.3.26

package/docs/mcp.md

@@ -36,8 +36,8 @@ after 30 seconds.
 
 The ACL the gateway computes for `/soajs/acl`: the user's ACL and packages, else the tenant
 application's, else the package's. For each service, the latest allowed version; a restricted
-service shows only the APIs its ACL lists, per method. To give agents a different list from your
-apps, give the agent its own application, key and package in the tenant.
+service shows only the APIs its ACL lists, per method, and none when it lists none. To give agents
+a different list from your apps, give the agent its own application, key and package in the tenant.
 
 A tool call is checked again by the gateway like any API call, so a tool shown by mistake is still
 refused.
@@ -63,6 +63,54 @@ Off unless enabled, in the `gateway` custom registry (`registry.custom.gateway.v
 | `ttl` | `300000` | How long the tools of a service are cached, ms |
 | `timeout` | `10000` | Fetching tools and calling a tool, ms |
 | `maxTools` | `300` | At most this many tools in a list |
+| `auth` | — | The authorization flow, see below; without it, clients send the key and token themselves |
+
+### Authorization (`mcp.auth`)
+
+With `auth`, an MCP client connects with only the URL and the MCP authorization flow (OAuth 2.1,
+PKCE). The gateway is the protected resource; the authorization server (consent page, codes, tokens)
+is yours, named by its endpoints.
+
+```json
+"auth": {
+  "byProduct": {
+    "AGENTS": {
+      "authorization_endpoint": "https://portal.example.com/agents/authorize",
+      "token_endpoint": "https://api.example.com/agents/oauth/token?key=<key>",
+      "registration_endpoint": "https://api.example.com/agents/oauth/register?key=<key>"
+    }
+  },
+  "byHost": {
+    "api.client.example.com": { "authorization_endpoint": "...", "token_endpoint": "...", "registration_endpoint": "..." }
+  },
+  "default": { "authorization_endpoint": "...", "token_endpoint": "...", "registration_endpoint": "..." }
+}
+```
+
+- **Which endpoints:**
+  - `byProduct`, by the product of the key on the request (header or `?key=`);
+  - else `byHost`, by the host the request came in on (`X-Forwarded-Host`, else `Host`);
+  - else `default`.
+  - An entry needs `authorization_endpoint` and `token_endpoint`; `registration_endpoint` is optional.
+- **No token, or an unknown or expired one:** HTTP 401 with
+  `WWW-Authenticate: Bearer resource_metadata="<origin>/.well-known/oauth-protected-resource/soajs/mcp"`,
+  plus `?key=` when the client's URL had one. If nothing matches the request, the answer is as without
+  `auth`.
+- **`GET /.well-known/oauth-protected-resource[/soajs/mcp]`:** `resource`, `authorization_servers`
+  (the issuer `<origin>/soajs/mcp/auth/<product/CODE|host|default>`) and `bearer_methods_supported`.
+- **`GET /.well-known/oauth-authorization-server/<issuer path>`** (also `openid-configuration`):
+  - the issuer and the entry's endpoints;
+  - `code` with PKCE `S256`;
+  - grant `authorization_code`;
+  - token endpoint auth `none`.
+- **The key from the token.** A client sends no key header. When the token is restricted to one key
+  (`user.restrictedTo.key`):
+  - `/soajs/mcp` without a key uses that key;
+  - with a key of the same product (a product URL), the token's key replaces it;
+  - a key of another product is kept, and the token is refused on it (147).
+- **The issued token:** the token endpoint should return one restricted to the agent's tenant,
+  product, package and key, e.g. from soajs.oauth `POST /restricted/token/guest`. The gateway then
+  checks it on every call like any restricted token.
 
 ## Protocol
 
@@ -91,8 +139,20 @@ streaming. `GET` answers 405.
 
 The arguments are split back into the path, the query string and the body, and the call goes to
 `http://127.0.0.1:<gateway port>/<service>/v<version><path>` with the caller's `key`,
-`authorization` / `access_token`, `device-id`, `user-agent`, `accept-language` and
-`idempotency-key` headers and `x-forwarded-for`.
+`authorization` / `access_token`, `device-id`, `user-agent` and `accept-language` headers and
+`x-forwarded-for`.
+
+**Idempotency.** An `Idempotency-Key` header on the MCP request is not forwarded: it is the same for
+every call of a connection, so the API would answer every later call with the first call's response.
+Instead:
+
+- A tool whose API has idempotency on (`registry.custom.gateway.value.idempotency`, by service and
+  API) takes an optional `_idempotencyKey` argument: a UUID v4 that the agent sends again when it
+  retries the same call, so the call runs once. A value that isn't a UUID v4 is a tool error, and
+  nothing is called.
+- A POST, PUT or DELETE without one gets a new key, so an API that requires a key works, and two
+  calls are never taken for one.
+- An API with its own `_idempotencyKey` field keeps it as a field.
 
 - **Success:** `content` has the response `data` as JSON text, `structuredContent` the same when it
   is an object.
@@ -136,5 +196,6 @@ URL.
 | File | Role |
 |------|------|
 | `mw/soajsRoutes/mcp/index.js` | The route: JSON-RPC handling |
-| `mw/soajsRoutes/mcp/tools.js` | Tools: fetch, cache, ACL filter, naming, the call |
+| `mw/soajsRoutes/mcp/tools.js` | Tools: fetch, cache, ACL filter, naming, idempotency, the call |
+| `mw/soajsRoutes/mcp/auth.js` | Authorization: the 401, the well-known metadata, the key from the token |
 | `mw/soajsRoutes/lib/acl.js` | The ACL computation shared with `/soajs/acl` |

package/mw/idempotency/index.js

@@ -222,4 +222,6 @@ module.exports = function (configuration) {
 
 // Export model for external use
 module.exports.model = model;
+module.exports.matchesApi = matchesApi;
+module.exports.isValidUUIDv4 = isValidUUIDv4;
 module.exports.unlock = unlock;

package/mw/soajsRoutes/lib/acl.js

@@ -16,10 +16,12 @@ const coreLibs = require("soajs.core.libs");
  *
  * finalACL is, per service, the latest allowed version and, for a restricted service, the allowed
  * apis per method: {"urac": {"version": "3", "get": ["/user"]}}. A service that is not restricted
- * has only its version: all its apis are allowed.
+ * has only its version: all its apis are allowed. A restricted service that lists no apis also has
+ * only its version, so restricted says, per service, whether its latest allowed version is
+ * restricted.
  *
  * @param req
- * @returns {{acl: object|null, finalACL: object|null, packages: Array|null}}
+ * @returns {{acl: object|null, finalACL: object|null, restricted: object, packages: Array|null}}
  */
 function computeACL(req) {
 	let ACL = null;
@@ -42,6 +44,7 @@ function computeACL(req) {
 		ACL = coreLibs.utils.cloneObj(req.soajs.controller.serviceParams.packObj.acl);
 	}
 	let finalACL = null;
+	let restricted = {};
 	if (ACL) {
 		finalACL = {};
 		for (let s in ACL) {
@@ -58,7 +61,8 @@ function computeACL(req) {
 						finalACL[s] = {
 							"version": v
 						};
-						if (sv.apisPermission && sv.apisPermission === "restricted") {
+						restricted[s] = !!(sv.apisPermission && sv.apisPermission === "restricted");
+						if (restricted[s]) {
 							for (let m in sv) {
 								if (sv.hasOwnProperty(m)) {
 									if (m !== "access" && m !== "apisPermission") {
@@ -91,6 +95,7 @@ function computeACL(req) {
 	return {
 		"acl": ACL,
 		"finalACL": finalACL,
+		"restricted": restricted,
 		"packages": ALLOWED_PACKAGES
 	};
 }

package/mw/soajsRoutes/mcp/auth.js

@@ -0,0 +1,331 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+/**
+ * The MCP authorization flow (OAuth 2.1, RFC 9728, RFC 8414) for /soajs/mcp, on only when
+ * registry.custom.gateway.value.mcp.auth is set. The gateway is the protected resource: it answers a
+ * call without a valid token with 401 and the address of its metadata, and serves the metadata. The
+ * authorization server (consent page, codes, PKCE, tokens) is not the gateway, the metadata points
+ * at the endpoints the configuration names.
+ *
+ * Which endpoints: by the product of the key on the request (auth.byProduct), else by the host the
+ * request came in on (auth.byHost), else auth.default.
+ *
+ * An MCP client sends the token only, no key. A token restricted to one key (user.restrictedTo.key)
+ * brings it: /soajs/mcp without a key uses it, and with a key of the same product the token's key
+ * replaces it.
+ */
+
+const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
+
+const MCP_PATH = "/soajs/mcp";
+const ISSUER_PATH = MCP_PATH + "/auth/";
+const PRM_PATH = "/.well-known/oauth-protected-resource";
+const AS_PATHS = ["/.well-known/oauth-authorization-server", "/.well-known/openid-configuration"];
+
+/**
+ * The auth configuration: registry.custom.gateway.value.mcp.auth, when mcp is enabled.
+ */
+function getAuthConfig(req) {
+	let mcp = get(["soajs", "registry", "custom", "gateway", "value", "mcp"], req);
+	if (!mcp || mcp.enabled !== true || !mcp.auth || typeof mcp.auth !== "object") {
+		return null;
+	}
+	return mcp.auth;
+}
+
+/**
+ * An entry is usable when it names the authorization and token endpoints.
+ */
+function usable(entry) {
+	return !!entry && typeof entry === "object" &&
+		typeof entry.authorization_endpoint === "string" && entry.authorization_endpoint !== "" &&
+		typeof entry.token_endpoint === "string" && entry.token_endpoint !== "";
+}
+
+function firstValue(header) {
+	return (typeof header === "string") ? header.split(",")[0].trim() : null;
+}
+
+/**
+ * The host the request came in on, and the origin the client sees.
+ */
+function requestHost(req) {
+	return (firstValue(req.headers["x-forwarded-host"]) || req.headers.host || "").toLowerCase();
+}
+
+function requestOrigin(req) {
+	let proto = firstValue(req.headers["x-forwarded-proto"]);
+	if (!proto) {
+		proto = (req.socket && req.socket.encrypted) ? "https" : "http";
+	}
+	return proto + "://" + requestHost(req);
+}
+
+function ownEntry(map, name) {
+	if (!map || typeof map !== "object" || !name || !Object.hasOwnProperty.call(map, name)) {
+		return null;
+	}
+	return usable(map[name]) ? map[name] : null;
+}
+
+/**
+ * The entry for a request: by the product of its key, by its host, the default.
+ *
+ * @param cb (match) match: {id, entry} or null
+ */
+function resolveEntry(req, provision, auth, key, cb) {
+	let byHost = () => {
+		let entry = ownEntry(auth.byHost, requestHost(req));
+		if (entry) {
+			return cb({"id": "host", "entry": entry});
+		}
+		if (usable(auth.default)) {
+			return cb({"id": "default", "entry": auth.default});
+		}
+		return cb(null);
+	};
+	if (!key || !auth.byProduct) {
+		return byHost();
+	}
+	provision.getExternalKeyData(key, get(["soajs", "registry", "serviceConfig", "key"], req), (err, keyObj) => {
+		let product = get(["application", "product"], keyObj);
+		let entry = ownEntry(auth.byProduct, product);
+		if (entry) {
+			return cb({"id": "product/" + product, "entry": entry});
+		}
+		return byHost();
+	});
+}
+
+/**
+ * The entry an issuer id names, as resolveEntry gave it.
+ */
+function entryById(req, auth, id) {
+	if (id === "host") {
+		return ownEntry(auth.byHost, requestHost(req));
+	}
+	if (id === "default") {
+		return usable(auth.default) ? auth.default : null;
+	}
+	if (id.indexOf("product/") === 0) {
+		return ownEntry(auth.byProduct, id.substring("product/".length));
+	}
+	return null;
+}
+
+function issuer(req, id) {
+	return requestOrigin(req) + ISSUER_PATH + id;
+}
+
+function sendJSON(res, status, body, headers) {
+	let text = JSON.stringify(body);
+	res.writeHead(status, Object.assign({
+		"Content-Type": "application/json",
+		"Content-Length": Buffer.byteLength(text),
+		"Cache-Control": "no-store"
+	}, headers || {}));
+	res.end(text);
+}
+
+/**
+ * 401 with the address of the protected resource metadata. A key the client put in the url stays
+ * in it, the metadata is then chosen by the same product.
+ */
+function challenge(req, res, queryKey, invalidToken) {
+	let metadata = requestOrigin(req) + PRM_PATH + MCP_PATH + (queryKey ? "?key=" + encodeURIComponent(queryKey) : "");
+	let value = 'Bearer resource_metadata="' + metadata + '"';
+	if (invalidToken) {
+		value += ', error="invalid_token"';
+	}
+	sendJSON(res, 401, {
+		"error": invalidToken ? "invalid_token" : "unauthorized",
+		"error_description": invalidToken ? "The access token is invalid or has expired." : "Authorization required."
+	}, {
+		"WWW-Authenticate": value,
+		"Access-Control-Expose-Headers": "WWW-Authenticate"
+	});
+}
+
+function queryOf(url) {
+	try {
+		return new URL(url, "http://localhost").searchParams;
+	} catch (e) {
+		return new URLSearchParams();
+	}
+}
+
+function pathOf(url) {
+	let i = url.indexOf("?");
+	return (i === -1) ? url : url.substring(0, i);
+}
+
+/**
+ * The token of a request: the access_token header or query, or Authorization: Bearer.
+ */
+function tokenOf(req) {
+	if (req.query && req.query.access_token) {
+		return req.query.access_token;
+	}
+	if (req.headers.access_token) {
+		return req.headers.access_token;
+	}
+	let matches = (req.headers.authorization || "").match(/^Bearer\s+(\S+)/i);
+	return matches ? matches[1] : null;
+}
+
+/**
+ * The key a token is restricted to, when it is exactly one.
+ */
+function tokenKey(record) {
+	let key = get(["user", "restrictedTo", "key"], record);
+	if (typeof key === "string" && key !== "") {
+		return key;
+	}
+	if (Array.isArray(key) && key.length === 1 && typeof key[0] === "string" && key[0] !== "") {
+		return key[0];
+	}
+	return null;
+}
+
+/**
+ * GET /.well-known/oauth-protected-resource[/soajs/mcp] and
+ * GET /.well-known/{oauth-authorization-server|openid-configuration}/soajs/mcp/auth/<id>,
+ * before the url mw, which knows no service named .well-known.
+ */
+function wellKnown(configuration) {
+	let provision = configuration.provision;
+
+	return (req, res, next) => {
+		let path = pathOf(req.url || "");
+		if (path.indexOf("/.well-known/") !== 0) {
+			return next();
+		}
+		let auth = getAuthConfig(req);
+		if (!auth || (req.method !== "GET" && req.method !== "HEAD")) {
+			return next();
+		}
+		if (path === PRM_PATH || path === PRM_PATH + MCP_PATH) {
+			let queryKey = queryOf(req.url).get("key");
+			return resolveEntry(req, provision, auth, queryKey, (match) => {
+				if (!match) {
+					return next();
+				}
+				return sendJSON(res, 200, {
+					"resource": requestOrigin(req) + MCP_PATH + (queryKey ? "?key=" + encodeURIComponent(queryKey) : ""),
+					"authorization_servers": [issuer(req, match.id)],
+					"bearer_methods_supported": ["header"]
+				});
+			});
+		}
+		for (let i = 0; i < AS_PATHS.length; i++) {
+			let prefix = AS_PATHS[i] + ISSUER_PATH;
+			if (path.indexOf(prefix) === 0) {
+				let id = decodeURIComponent(path.substring(prefix.length));
+				let entry = entryById(req, auth, id);
+				if (!entry) {
+					return next();
+				}
+				let metadata = {
+					"issuer": issuer(req, id),
+					"authorization_endpoint": entry.authorization_endpoint,
+					"token_endpoint": entry.token_endpoint,
+					"response_types_supported": ["code"],
+					"grant_types_supported": ["authorization_code"],
+					"code_challenge_methods_supported": ["S256"],
+					"token_endpoint_auth_methods_supported": ["none"]
+				};
+				if (typeof entry.registration_endpoint === "string" && entry.registration_endpoint !== "") {
+					metadata.registration_endpoint = entry.registration_endpoint;
+				}
+				return sendJSON(res, 200, metadata);
+			}
+		}
+		return next();
+	};
+}
+
+/**
+ * /soajs/mcp, after the url mw and before the key mw: the 401 when the caller must authorize, and
+ * the key from a token restricted to one key.
+ */
+function authorize(configuration) {
+	let provision = configuration.provision;
+
+	return (req, res, next) => {
+		let info = get(["soajs", "controller", "serviceParams", "serviceInfo"], req);
+		if (!info || info[1] !== "soajs" || info[2] !== "mcp") {
+			return next();
+		}
+		let auth = getAuthConfig(req);
+		let mcp = get(["soajs", "registry", "custom", "gateway", "value", "mcp"], req);
+		if (!mcp || mcp.enabled !== true) {
+			return next();
+		}
+		let queryKey = (req.query && req.query.key) || null;
+		let askToAuthorize = (invalidToken) => {
+			if (!auth) {
+				return next();
+			}
+			resolveEntry(req, provision, auth, req.headers.key, (match) => {
+				if (!match) {
+					return next();
+				}
+				return challenge(req, res, queryKey, invalidToken);
+			});
+		};
+
+		let token = tokenOf(req);
+		if (!token) {
+			return askToAuthorize(false);
+		}
+		provision.oauthModel.getAccessToken(token, (err, record) => {
+			if (err) {
+				req.soajs.log.error("MCP: unable to read the access token: " + err.message);
+				return next();
+			}
+			if (!record || (record.expires && new Date(record.expires).getTime() < Date.now())) {
+				return askToAuthorize(true);
+			}
+			let restrictedKey = tokenKey(record);
+			if (!restrictedKey || req.headers.key === restrictedKey) {
+				return next();
+			}
+			if (!req.headers.key) {
+				req.headers.key = restrictedKey;
+				return next();
+			}
+			//NOTE: the url names a product, the token a key of it: the token's key. A key of another
+			//		product is left as is, the oauth mw refuses the token on it (147).
+			let keyConfig = get(["soajs", "registry", "serviceConfig", "key"], req);
+			provision.getExternalKeyData(req.headers.key, keyConfig, (err, given) => {
+				provision.getExternalKeyData(restrictedKey, keyConfig, (err, own) => {
+					let givenProduct = get(["application", "product"], given);
+					let ownProduct = get(["application", "product"], own);
+					if (givenProduct && givenProduct === ownProduct) {
+						req.headers.key = restrictedKey;
+					}
+					return next();
+				});
+			});
+		});
+	};
+}
+
+module.exports = {
+	getAuthConfig,
+	resolveEntry,
+	entryById,
+	requestOrigin,
+	tokenOf,
+	tokenKey,
+	wellKnown,
+	authorize
+};

package/mw/soajsRoutes/mcp/tools.js

@@ -18,10 +18,12 @@
  */
 
 const http = require("http");
+const crypto = require("crypto");
 const yaml = require("js-yaml");
 const coreLibs = require("soajs.core.libs");
 
 const aclLib = require("../lib/acl.js");
+const idempotency = require("../../idempotency/index.js");
 
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 
@@ -37,7 +39,13 @@ const NAME_MAX = 64;
 // service@version -> {expires, tools}
 let cache = {};
 
-const HEADERS_TO_FORWARD = ['key', 'authorization', 'access_token', 'device-id', 'user-agent', 'accept-language', 'idempotency-key'];
+//NOTE: no idempotency-key, a header on the mcp request is the same for every tool call of the
+//		connection, the api would replay the first answer to every later call. A tool call carries
+//		its own key, see IDEMPOTENCY_ARG.
+const HEADERS_TO_FORWARD = ['key', 'authorization', 'access_token', 'device-id', 'user-agent', 'accept-language'];
+
+// the argument a tool call gives its idempotency key in, on the tools whose api has idempotency on
+const IDEMPOTENCY_ARG = "_idempotencyKey";
 
 /**
  * The mcp configuration: registry.custom.gateway.value.mcp
@@ -224,11 +232,10 @@ function toolName(serviceName, tool, taken) {
 }
 
 /**
- * Whether the caller's ACL allows the api: a restricted service lists its apis per method.
+ * Whether the caller's ACL allows the api: a restricted service allows only the apis it lists per
+ * method, none when it lists none.
  */
-function allowed(serviceACL, tool) {
-	let methods = ["get", "post", "put", "delete", "del"];
-	let restricted = methods.some((m) => Array.isArray(serviceACL[m]));
+function allowed(serviceACL, tool, restricted) {
 	if (!restricted) {
 		return true;
 	}
@@ -236,6 +243,33 @@ function allowed(serviceACL, tool) {
 	return Array.isArray(list) && list.indexOf(tool.v) !== -1;
 }
 
+/**
+ * Whether the gateway applies idempotency to the api of a tool, as mw/idempotency decides it for
+ * the api call: registry.custom.gateway.value.idempotency, by service and api. A get never.
+ */
+function idempotent(req, serviceName, tool) {
+	if (tool.m === "get") {
+		return false;
+	}
+	let services = get(["soajs", "registry", "custom", "gateway", "value", "idempotency", "services"], req);
+	let serviceConfig = services ? services[serviceName] : null;
+	if (!serviceConfig || !serviceConfig.enabled) {
+		return false;
+	}
+	if (serviceConfig.apis && !idempotency.matchesApi(tool.m.toUpperCase(), tool.v, serviceConfig.apis)) {
+		return false;
+	}
+	return true;
+}
+
+/**
+ * Whether a tool call can carry its idempotency key in IDEMPOTENCY_ARG: not a get, and not an
+ * api with a field of that name.
+ */
+function takesIdempotencyArg(tool) {
+	return tool.m !== "get" && !Object.hasOwnProperty.call(tool.fields || {}, IDEMPOTENCY_ARG);
+}
+
 /**
  * The tools the caller may use.
  *
@@ -253,12 +287,22 @@ function listTools(req, configuration, config) {
 			let taken = {};
 			services.forEach((serviceName, i) => {
 				lists[i].forEach((tool) => {
-					if (tools.length >= config.maxTools || !allowed(finalACL[serviceName], tool)) {
+					if (tools.length >= config.maxTools || !allowed(finalACL[serviceName], tool, computed.restricted[serviceName])) {
 						return;
 					}
 					let name = toolName(serviceName, tool, taken);
 					let description = tool.l || (tool.m.toUpperCase() + " " + tool.v);
 					let inputSchema = (tool.inputSchema && tool.inputSchema.type === "object") ? tool.inputSchema : {"type": "object", "properties": {}};
+					if (takesIdempotencyArg(tool) && idempotent(req, serviceName, tool)) {
+						//NOTE: copies, the tool is shared through the cache
+						let properties = Object.assign({}, inputSchema.properties || {});
+						properties[IDEMPOTENCY_ARG] = {
+							"type": "string",
+							"format": "uuid",
+							"description": "Optional UUID v4. Send the same value when retrying this exact call, so it runs once."
+						};
+						inputSchema = Object.assign({}, inputSchema, {"properties": properties});
+					}
 					tools.push({
 						"name": name,
 						"title": description,
@@ -372,6 +416,22 @@ function callTool(req, configuration, config, entry, args) {
 			headers[h] = req.headers[h];
 		}
 	});
+	if (takesIdempotencyArg(entry.tool)) {
+		//NOTE: a write without a key gets a new one: an api that requires a key works, and two
+		//		calls are never taken for one. Only a key the agent sends again makes a retry run once.
+		let given = (args && typeof args === "object" && !Array.isArray(args)) ? args[IDEMPOTENCY_ARG] : undefined;
+		if (given !== undefined && given !== null && given !== "") {
+			if (typeof given !== "string" || !idempotency.isValidUUIDv4(given)) {
+				return Promise.resolve({
+					"content": [{"type": "text", "text": IDEMPOTENCY_ARG + " must be a UUID v4."}],
+					"isError": true
+				});
+			}
+			headers["idempotency-key"] = given;
+		} else {
+			headers["idempotency-key"] = crypto.randomUUID();
+		}
+	}
 	if (!headers.access_token && req.query && req.query.access_token) {
 		headers.access_token = req.query.access_token;
 	}
@@ -428,6 +488,8 @@ module.exports = {
 	serviceTools,
 	toolName,
 	allowed,
+	idempotent,
+	takesIdempotencyArg,
 	listTools,
 	buildCall,
 	callTool,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "soajs.controller",
   "description": "soajs multi tenant API gateway",
-  "version": "4.3.24",
+  "version": "4.3.26",
   "author": {
     "name": "soajs team",
     "email": "team@soajs.org"

package/server/controller.js

@@ -33,6 +33,7 @@ const awareness_mw = require('../mw/awareness/index');
 const awarenessEnv_mw = require("../mw/awarenessEnv/index");
 const url_mw = require("../mw/url/index");
 const key_mw = require("../mw/key/index");
+const mcpAuth_mw = require("../mw/soajsRoutes/mcp/auth");
 const keyACL_mw = require("../mw/keyACL/index");
 const gotoService_mw = require("../mw/gotoService/index");
 const mt_mw = require("../mw/mt/index");
@@ -96,6 +97,9 @@ Controller.prototype.init = function (callback) {
 			app.use(ip2ban_mw());
 			app.use(maintenanceMode_mw()); //NOTE: this is for global maintenance mode
 
+			//NOTE: before the url mw, which knows no service named .well-known
+			app.use(mcpAuth_mw.wellKnown({ "provision": provision }));
+
 			app.use(url_mw({ "core": core }));
 			
 			log.info("Loading Provision ...");
@@ -141,6 +145,9 @@ Controller.prototype.init = function (callback) {
 
 					// app.use(url_mw({ "core": core }));
 
+					//NOTE: before the key mw, a token restricted to one key brings the key of /soajs/mcp
+					app.use(mcpAuth_mw.authorize({ "provision": provision }));
+
 					app.use(key_mw({ "provision": provision, "regEnvironment": regEnvironment }));
 
 					//TODO: add maintenance by tenant here req.soajs.controller.serviceParams.keyObj.config

package/test/unit/index.js

@@ -38,6 +38,7 @@ describe("Starting Gateway Unit test", () => {
         require("./mw/soajs/index.js");
         require("./mw/soajsRoutes/keyACL.js");
         require("./mw/soajsRoutes/mcp.js");
+        require("./mw/soajsRoutes/mcpAuth.js");
         require("./mw/traffic/index.js");
         require("./mw/idempotency/index.js");
         require("./mw/idempotency/memory.js");

package/test/unit/mw/soajsRoutes/mcp.js

@@ -216,14 +216,35 @@ describe("Unit test for: mw - soajsRoutes mcp", () => {
 
 	describe("allowed", () => {
 		it("a service that is not restricted allows all its apis", () => {
-			assert.ok(tools.allowed({version: "1"}, {m: "delete", v: "/x"}));
+			assert.ok(tools.allowed({version: "1"}, {m: "delete", v: "/x"}, false));
 		});
 		it("a restricted service allows the apis it lists", () => {
 			let acl = {version: "1", get: ["/users"], del: ["/user/:id"]};
-			assert.ok(tools.allowed(acl, {m: "get", v: "/users"}));
-			assert.ok(!tools.allowed(acl, {m: "get", v: "/user/:id"}));
-			assert.ok(tools.allowed(acl, {m: "delete", v: "/user/:id"}));
-			assert.ok(!tools.allowed(acl, {m: "post", v: "/users"}));
+			assert.ok(tools.allowed(acl, {m: "get", v: "/users"}, true));
+			assert.ok(!tools.allowed(acl, {m: "get", v: "/user/:id"}, true));
+			assert.ok(tools.allowed(acl, {m: "delete", v: "/user/:id"}, true));
+			assert.ok(!tools.allowed(acl, {m: "post", v: "/users"}, true));
+		});
+		it("a restricted service that lists no apis allows none", () => {
+			assert.ok(!tools.allowed({version: "1"}, {m: "get", v: "/users"}, true));
+		});
+	});
+
+	describe("idempotent", () => {
+		let reqWith = (idempotency) => {
+			let req = makeReq();
+			req.soajs.registry.custom.gateway.value.idempotency = idempotency;
+			return req;
+		};
+		it("off without configuration, for a get, a disabled service or an api not listed", () => {
+			assert.strictEqual(tools.idempotent(makeReq(), "urac", coreTools[1]), false);
+			assert.strictEqual(tools.idempotent(reqWith({services: {urac: {enabled: true}}}), "urac", coreTools[0]), false);
+			assert.strictEqual(tools.idempotent(reqWith({services: {urac: {enabled: false}}}), "urac", coreTools[1]), false);
+			assert.strictEqual(tools.idempotent(reqWith({services: {urac: {enabled: true, apis: ["DELETE /user/:id"]}}}), "urac", coreTools[1]), false);
+		});
+		it("on for every write of an enabled service, or the apis it lists", () => {
+			assert.strictEqual(tools.idempotent(reqWith({services: {urac: {enabled: true}}}), "urac", coreTools[1]), true);
+			assert.strictEqual(tools.idempotent(reqWith({services: {urac: {enabled: true, apis: ["POST /user/:id"]}}}), "urac", coreTools[1]), true);
 		});
 	});
 
@@ -342,6 +363,32 @@ describe("Unit test for: mw - soajsRoutes mcp", () => {
 			}).catch(done);
 		});
 
+		it("listTools: the tools whose api has idempotency on take _idempotencyKey, the cache is not changed", (done) => {
+			let req = makeReq({services: services(), acl: {urac: {"1": {access: true}}}});
+			req.soajs.registry.custom.gateway.value.idempotency = {services: {urac: {enabled: true, apis: ["POST /user/:id"]}}};
+			tools.listTools(req, configuration, config).then((list) => {
+				let byRoute = {};
+				list.tools.forEach((t) => {
+					byRoute[t.description.split("[urac: ")[1]] = t;
+				});
+				let post = byRoute["POST /user/:id]"];
+				assert.deepStrictEqual(post.inputSchema.properties._idempotencyKey.type, "string");
+				assert.deepStrictEqual(post.inputSchema.required, ["id"]);
+				assert.ok(!byRoute["DELETE /user/:id]"].inputSchema.properties._idempotencyKey);
+				assert.ok(!byRoute["GET /users]"].inputSchema.properties._idempotencyKey);
+				assert.ok(!coreTools[1].inputSchema.properties._idempotencyKey);
+				done();
+			}).catch(done);
+		});
+
+		it("listTools: a restricted service that lists no apis has no tools", (done) => {
+			let acl = {urac: {"1": {access: true, apisPermission: "restricted", post: {apis: {}}}}};
+			tools.listTools(makeReq({services: services(), acl: acl}), configuration, config).then((list) => {
+				assert.deepStrictEqual(list.tools, []);
+				done();
+			}).catch(done);
+		});
+
 		describe("callTool", () => {
 			let gateway = null;
 			let gatewayPort = null;
@@ -379,7 +426,9 @@ describe("Unit test for: mw - soajsRoutes mcp", () => {
 					assert.strictEqual(received.headers.authorization, "Bearer abc");
 					assert.strictEqual(received.headers.access_token, "tok");
 					assert.strictEqual(received.headers["device-id"], "d1");
-					assert.strictEqual(received.headers["idempotency-key"], "k1");
+					// the mcp request's own key is not forwarded, a write gets a new one
+					assert.notStrictEqual(received.headers["idempotency-key"], "k1");
+					assert.ok(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(received.headers["idempotency-key"]));
 					assert.strictEqual(received.headers["x-forwarded-for"], "10.0.0.5");
 					assert.ok(!received.headers.cookie);
 					assert.deepStrictEqual(result, {
@@ -391,6 +440,41 @@ describe("Unit test for: mw - soajsRoutes mcp", () => {
 				}).catch(done);
 			});
 
+			it("idempotency: a key the agent gives is sent, a bad one is refused, a get has none", (done) => {
+				reply = {status: 200, body: {result: true, data: {}}};
+				let key = "3b241101-e2bb-4255-8caf-4136c566a962";
+				received = null;
+				tools.callTool(makeReq({gatewayPort: gatewayPort}), configuration, config, entry, {id: "u1", _idempotencyKey: key}).then(() => {
+					assert.strictEqual(received.headers["idempotency-key"], key);
+					// the key is not an api field, it is not in the body
+					assert.strictEqual(received.body, "");
+					received = null;
+					return tools.callTool(makeReq({gatewayPort: gatewayPort}), configuration, config, entry, {id: "u1", _idempotencyKey: "k1"});
+				}).then((result) => {
+					assert.deepStrictEqual(result, {"content": [{"type": "text", "text": "_idempotencyKey must be a UUID v4."}], "isError": true});
+					assert.strictEqual(received, null);
+					return tools.callTool(makeReq({gatewayPort: gatewayPort, headers: {"idempotency-key": key}}), configuration, config, {service: "urac", version: "1", tool: coreTools[0]}, {});
+				}).then(() => {
+					assert.ok(!received.headers["idempotency-key"]);
+					done();
+				}).catch(done);
+			});
+
+			it("idempotency: an api with its own _idempotencyKey field keeps it", (done) => {
+				reply = {status: 200, body: {result: true, data: {}}};
+				let tool = {
+					"m": "post", "v": "/thing", "l": "Thing",
+					"inputSchema": {"type": "object", "properties": {"_idempotencyKey": {"type": "string"}}},
+					"fields": {"_idempotencyKey": {"in": "body", "path": ["_idempotencyKey"]}}
+				};
+				tools.callTool(makeReq({gatewayPort: gatewayPort}), configuration, config, {service: "urac", version: "1", tool: tool}, {_idempotencyKey: "mine"}).then((result) => {
+					assert.strictEqual(result.isError, false);
+					assert.ok(!received.headers["idempotency-key"]);
+					assert.deepStrictEqual(JSON.parse(received.body), {_idempotencyKey: "mine"});
+					done();
+				}).catch(done);
+			});
+
 			it("a get has no body, an array answer has no structuredContent", (done) => {
 				reply = {status: 200, body: {result: true, data: [1, 2]}};
 				let req = makeReq({gatewayPort: gatewayPort});
@@ -568,5 +652,16 @@ describe("Unit test for: mw - soajsRoutes mcp", () => {
 			req = makeReq({acl: {urac: {"1": {access: true, apisPermission: "restricted", get: [{group: "a", apis: {"/x": {}}}, {group: "b", apis: {"/y": {}, "/x": {}}}]}}}});
 			assert.deepStrictEqual(aclLib.computeACL(req).finalACL, {"urac": {"version": "1", "get": ["/x", "/y"]}});
 		});
+		it("restricted follows the latest allowed version", () => {
+			let req = makeReq({
+				acl: {
+					urac: {"1": {access: true, apisPermission: "restricted", get: {apis: {"/x": {}}}}, "2": {access: true}},
+					oauth: {"1": {access: true, apisPermission: "restricted", post: {apis: {}}}}
+				}
+			});
+			let computed = aclLib.computeACL(req);
+			assert.deepStrictEqual(computed.finalACL, {"urac": {"version": "2"}, "oauth": {"version": "1"}});
+			assert.deepStrictEqual(computed.restricted, {"urac": false, "oauth": true});
+		});
 	});
 });

package/test/unit/mw/soajsRoutes/mcpAuth.js

@@ -0,0 +1,280 @@
+"use strict";
+
+const helper = require("../../../helper.js");
+const auth = helper.requireModule('./mw/soajsRoutes/mcp/auth');
+const assert = require('assert');
+
+const KEYS = {
+	"agentsKey": {application: {product: "AGENTS", package: "AGENTS_ROLE"}},
+	"agentKey": {application: {product: "AGENTS", package: "AGENTS_ROLE"}},
+	"appKey": {application: {product: "AVAPP", package: "AVAPP_DEFAULT"}}
+};
+
+const TOKENS = {
+	"agentToken": {user: {restrictedTo: {tenant: "t1", key: "agentKey"}}, expires: new Date(Date.now() + 60000)},
+	"listToken": {user: {restrictedTo: {tenant: "t1", key: ["agentKey"]}}, expires: new Date(Date.now() + 60000)},
+	"userToken": {user: {}, expires: new Date(Date.now() + 60000)},
+	"oldToken": {user: {restrictedTo: {key: "agentKey"}}, expires: new Date(Date.now() - 1000)}
+};
+
+const provision = {
+	getExternalKeyData: (key, keyConfig, cb) => {
+		return KEYS[key] ? cb(null, KEYS[key]) : cb(new Error("no key"), null);
+	},
+	oauthModel: {
+		getAccessToken: (token, cb) => {
+			if (token === "broken") {
+				return cb(new Error("db down"));
+			}
+			return cb(null, TOKENS[token] || null);
+		}
+	}
+};
+
+const ENTRY = (name) => {
+	return {
+		"authorization_endpoint": "https://portal." + name + "/agents/authorize",
+		"token_endpoint": "https://api." + name + "/agents/oauth/token?key=agentsKey",
+		"registration_endpoint": "https://api." + name + "/agents/oauth/register?key=agentsKey"
+	};
+};
+
+const AUTH = {
+	"byProduct": {"AGENTS": ENTRY("agents")},
+	"byHost": {"api.client.example": ENTRY("client")},
+	"default": ENTRY("default")
+};
+
+let makeReq = (opts) => {
+	opts = opts || {};
+	let mcp = (opts.mcp === undefined) ? {enabled: true, auth: AUTH} : opts.mcp;
+	let req = {
+		method: opts.method || "POST",
+		url: opts.url || "/soajs/mcp",
+		headers: Object.assign({host: "api.connect.example", "x-forwarded-proto": "https"}, opts.headers || {}),
+		query: opts.query || {},
+		soajs: {
+			log: {error: () => {
+			}},
+			registry: {
+				serviceConfig: {key: {}},
+				custom: {gateway: {value: {mcp: mcp}}}
+			},
+			controller: {
+				serviceParams: {serviceInfo: opts.serviceInfo || ["", "soajs", "mcp"]}
+			}
+		}
+	};
+	return req;
+};
+
+/**
+ * Runs a mw: cb(null) when it calls next, cb(answer) when it answers {status, headers, body}.
+ */
+let run = (mw, req, cb) => {
+	let res = {
+		writeHead: (status, headers) => {
+			res.status = status;
+			res.headers = headers;
+		},
+		end: (text) => {
+			cb({status: res.status, headers: res.headers, body: JSON.parse(text)});
+		}
+	};
+	mw(req, res, () => cb(null));
+};
+
+describe("Unit test for: mw - soajsRoutes mcp auth", () => {
+
+	describe("well-known metadata", () => {
+		let mw = auth.wellKnown({provision: provision});
+
+		it("not a well-known path, mcp off or no auth: next", (done) => {
+			run(mw, makeReq({method: "GET", url: "/urac/user"}), (answer) => {
+				assert.strictEqual(answer, null);
+				run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource", mcp: {enabled: true}}), (answer) => {
+					assert.strictEqual(answer, null);
+					run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource", mcp: {enabled: false, auth: AUTH}}), (answer) => {
+						assert.strictEqual(answer, null);
+						done();
+					});
+				});
+			});
+		});
+
+		it("protected resource: by the product of the key in the url", (done) => {
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource/soajs/mcp?key=agentsKey"}), (answer) => {
+				assert.strictEqual(answer.status, 200);
+				assert.deepStrictEqual(answer.body, {
+					"resource": "https://api.connect.example/soajs/mcp?key=agentsKey",
+					"authorization_servers": ["https://api.connect.example/soajs/mcp/auth/product/AGENTS"],
+					"bearer_methods_supported": ["header"]
+				});
+				done();
+			});
+		});
+
+		it("protected resource: by host, then the default, a product with no entry falls back", (done) => {
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource", headers: {"x-forwarded-host": "api.client.example, proxy"}}), (answer) => {
+				assert.deepStrictEqual(answer.body.authorization_servers, ["https://api.client.example/soajs/mcp/auth/host"]);
+				assert.strictEqual(answer.body.resource, "https://api.client.example/soajs/mcp");
+				run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource/soajs/mcp?key=appKey"}), (answer) => {
+					assert.deepStrictEqual(answer.body.authorization_servers, ["https://api.connect.example/soajs/mcp/auth/default"]);
+					done();
+				});
+			});
+		});
+
+		it("protected resource: nothing configured for the request: next", (done) => {
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource", mcp: {enabled: true, auth: {byHost: {"other": ENTRY("x")}}}}), (answer) => {
+				assert.strictEqual(answer, null);
+				done();
+			});
+		});
+
+		it("authorization server: the entry the issuer names, in both well-known forms", (done) => {
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-authorization-server/soajs/mcp/auth/product/AGENTS"}), (answer) => {
+				assert.strictEqual(answer.status, 200);
+				assert.deepStrictEqual(answer.body, {
+					"issuer": "https://api.connect.example/soajs/mcp/auth/product/AGENTS",
+					"authorization_endpoint": "https://portal.agents/agents/authorize",
+					"token_endpoint": "https://api.agents/agents/oauth/token?key=agentsKey",
+					"response_types_supported": ["code"],
+					"grant_types_supported": ["authorization_code"],
+					"code_challenge_methods_supported": ["S256"],
+					"token_endpoint_auth_methods_supported": ["none"],
+					"registration_endpoint": "https://api.agents/agents/oauth/register?key=agentsKey"
+				});
+				run(mw, makeReq({method: "GET", url: "/.well-known/openid-configuration/soajs/mcp/auth/default"}), (answer) => {
+					assert.strictEqual(answer.body.authorization_endpoint, "https://portal.default/agents/authorize");
+					done();
+				});
+			});
+		});
+
+		it("authorization server: an unknown issuer, an inherited name or a post: next", (done) => {
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-authorization-server/soajs/mcp/auth/product/OTHER"}), (answer) => {
+				assert.strictEqual(answer, null);
+				run(mw, makeReq({method: "GET", url: "/.well-known/oauth-authorization-server/soajs/mcp/auth/product/constructor"}), (answer) => {
+					assert.strictEqual(answer, null);
+					run(mw, makeReq({method: "POST", url: "/.well-known/oauth-authorization-server/soajs/mcp/auth/default"}), (answer) => {
+						assert.strictEqual(answer, null);
+						done();
+					});
+				});
+			});
+		});
+
+		it("an entry without the token endpoint is not used", (done) => {
+			let mcp = {enabled: true, auth: {default: {authorization_endpoint: "https://p/a"}}};
+			run(mw, makeReq({method: "GET", url: "/.well-known/oauth-protected-resource", mcp: mcp}), (answer) => {
+				assert.strictEqual(answer, null);
+				done();
+			});
+		});
+	});
+
+	describe("authorize", () => {
+		let mw = auth.authorize({provision: provision});
+
+		it("another route, or mcp off: next, untouched", (done) => {
+			let req = makeReq({serviceInfo: ["", "urac", "user"]});
+			run(mw, req, (answer) => {
+				assert.strictEqual(answer, null);
+				run(mw, makeReq({mcp: {enabled: false, auth: AUTH}}), (answer) => {
+					assert.strictEqual(answer, null);
+					done();
+				});
+			});
+		});
+
+		it("no token and no auth configured: next, as before", (done) => {
+			run(mw, makeReq({mcp: {enabled: true}}), (answer) => {
+				assert.strictEqual(answer, null);
+				done();
+			});
+		});
+
+		it("no token: 401 with the metadata address, the url key kept", (done) => {
+			run(mw, makeReq({headers: {key: "agentsKey"}, query: {key: "agentsKey"}}), (answer) => {
+				assert.strictEqual(answer.status, 401);
+				assert.strictEqual(answer.headers["WWW-Authenticate"],
+					'Bearer resource_metadata="https://api.connect.example/.well-known/oauth-protected-resource/soajs/mcp?key=agentsKey"');
+				assert.strictEqual(answer.body.error, "unauthorized");
+				run(mw, makeReq(), (answer) => {
+					assert.strictEqual(answer.headers["WWW-Authenticate"],
+						'Bearer resource_metadata="https://api.connect.example/.well-known/oauth-protected-resource/soajs/mcp"');
+					done();
+				});
+			});
+		});
+
+		it("an unknown or expired token: 401 invalid_token", (done) => {
+			run(mw, makeReq({headers: {authorization: "Bearer nope"}}), (answer) => {
+				assert.strictEqual(answer.status, 401);
+				assert.ok(answer.headers["WWW-Authenticate"].indexOf('error="invalid_token"') !== -1);
+				run(mw, makeReq({headers: {access_token: "oldToken"}}), (answer) => {
+					assert.strictEqual(answer.body.error, "invalid_token");
+					done();
+				});
+			});
+		});
+
+		it("the token store fails: next, the oauth mw decides", (done) => {
+			run(mw, makeReq({query: {access_token: "broken"}}), (answer) => {
+				assert.strictEqual(answer, null);
+				done();
+			});
+		});
+
+		it("no key: the key the token is restricted to, a one-key list too", (done) => {
+			let req = makeReq({headers: {authorization: "Bearer agentToken"}});
+			run(mw, req, (answer) => {
+				assert.strictEqual(answer, null);
+				assert.strictEqual(req.headers.key, "agentKey");
+				let req2 = makeReq({headers: {authorization: "Bearer listToken"}});
+				run(mw, req2, () => {
+					assert.strictEqual(req2.headers.key, "agentKey");
+					done();
+				});
+			});
+		});
+
+		it("a key of the same product is replaced, of another product kept", (done) => {
+			let req = makeReq({headers: {authorization: "Bearer agentToken", key: "agentsKey"}});
+			run(mw, req, (answer) => {
+				assert.strictEqual(answer, null);
+				assert.strictEqual(req.headers.key, "agentKey");
+				let req2 = makeReq({headers: {authorization: "Bearer agentToken", key: "appKey"}});
+				run(mw, req2, () => {
+					assert.strictEqual(req2.headers.key, "appKey");
+					done();
+				});
+			});
+		});
+
+		it("a token with no restricted key leaves the request as it is", (done) => {
+			let req = makeReq({headers: {authorization: "Bearer userToken"}});
+			run(mw, req, (answer) => {
+				assert.strictEqual(answer, null);
+				assert.ok(!req.headers.key);
+				done();
+			});
+		});
+	});
+
+	describe("helpers", () => {
+		it("tokenKey: one key only", () => {
+			assert.strictEqual(auth.tokenKey({user: {restrictedTo: {key: "a"}}}), "a");
+			assert.strictEqual(auth.tokenKey({user: {restrictedTo: {key: ["a"]}}}), "a");
+			assert.strictEqual(auth.tokenKey({user: {restrictedTo: {key: ["a", "b"]}}}), null);
+			assert.strictEqual(auth.tokenKey({user: {}}), null);
+			assert.strictEqual(auth.tokenKey(null), null);
+		});
+		it("requestOrigin: forwarded proto and host, else the connection", () => {
+			let req = makeReq({headers: {"x-forwarded-proto": undefined}});
+			req.socket = {encrypted: false};
+			assert.strictEqual(auth.requestOrigin(req), "http://api.connect.example");
+		});
+	});
+});