soajs.controller 4.3.11 → 4.3.13

package/lib/logOnce.js

@@ -0,0 +1,45 @@
+'use strict';
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const reported = new Set();
+
+/**
+ * Log a message once per distinct key.
+ *
+ * Static misconfiguration is discovered inside the request path, so reporting it
+ * normally would repeat on every request for the lifetime of the process. The
+ * condition cannot resolve itself without a registry change, so the first report
+ * carries all the information the rest would.
+ *
+ * @param log    the soajs logger
+ * @param level  one of the logger levels, ex: error
+ * @param key    identifies the condition, repeats with the same key are dropped
+ * @param message
+ */
+function logOnce(log, level, key, message) {
+	if (reported.has(key)) {
+		return false;
+	}
+	if (!log || typeof log[level] !== "function") {
+		return false;
+	}
+	reported.add(key);
+	log[level](message);
+	return true;
+}
+
+/**
+ * Forget what has been reported. Used by the unit tests.
+ */
+function resetLogOnce() {
+	reported.clear();
+}
+
+module.exports = { logOnce, resetLogOnce };

package/mw/awareness/custom.js

@@ -145,7 +145,7 @@ let awareness_healthCheck = function (core, log) {
 
 					let stopLoggingAfter = registry.serviceConfig.awareness.maxLogCount;
 					if (statusObj.downCount <= stopLoggingAfter) {
-						log.warn("Self Awareness health check for service [" + sObj.name + "] for host [" + sObj.host + "] is NOT healthy");
+						log.error("Self Awareness health check for service [" + sObj.name + "] for host [" + sObj.host + "] is NOT healthy");
 					}
 					if (serviceAwarenessObj[sObj.name].healthy[sObj.version].indexOf(sObj.host) !== -1) {
 						//TODO: if we guarantee uniqueness we will not need the for loop
@@ -192,7 +192,7 @@ let awareness_healthCheck = function (core, log) {
 			}
 		}, function (err) {
 			if (err) {
-				log.warn('Unable to build awareness structure for services: ' + err);
+				log.error('Unable to build awareness structure for services: ' + err);
 			}
 		});
 	setTimeout(() => {

package/mw/awarenessEnv/custom.js

@@ -26,7 +26,7 @@ let fetchControllerHosts = function (core, log, next) {
 	registry = registryModule.get();
 	registryModule.loadOtherEnvControllerHosts(gatewayServiceName, function (error, hosts) {
 		if (error) {
-			log.warn("Failed to load controller hosts. reusing from previous load. Reason: " + error.message);
+			log.error("Failed to load controller hosts. reusing from previous load. Reason: " + error.message);
 		} else {
 			controllerHosts = hosts;
 		}
@@ -107,7 +107,7 @@ let awareness_healthCheck = function (core, log) {
 					// });
 				}, function (err) {
 					if (err) {
-						log.warn('Unable to build awareness ENV structure for controllers: ' + err);
+						log.error('Unable to build awareness ENV structure for controllers: ' + err);
 					}
 				});
 		}

package/mw/cache/index.js

@@ -12,6 +12,7 @@ const crypto = require('crypto');
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 const memory = require('./model/memory.js');
 const mongo = require('./model/mongo.js');
+const { logOnce } = require('../../lib/logOnce.js');
 
 let model = { memory, mongo };
 
@@ -102,7 +103,8 @@ module.exports = function (configuration) {
 
 		const modelType = cacheConfig.model || 'memory';
 		if (!model[modelType]) {
-			configuration.log.warn('Cache: Unknown model [' + modelType + ']. It can only be [memory || mongo]');
+			logOnce(configuration.log, "error", "cache_model_" + modelType,
+				'Cache: Unknown model [' + modelType + ']. It can only be [memory || mongo]. Caching is off.');
 			return next();
 		}
 
@@ -119,7 +121,8 @@ module.exports = function (configuration) {
 		let scope = apiConfig.scope;
 		if (scope !== 'tenant' && scope !== 'tenant_user') {
 			if (scope) {
-				configuration.log.warn('Cache: Unknown scope [' + scope + '] for API [' + method + ' ' + apiPath + ']. It can only be [tenant || tenant_user]. Falling back to default.');
+				logOnce(configuration.log, "error", "cache_scope_" + scope + "_" + method + "_" + apiPath,
+					'Cache: Unknown scope [' + scope + '] for API [' + method + ' ' + apiPath + ']. It can only be [tenant || tenant_user]. Falling back to default.');
 			}
 			scope = isAPIPublic ? 'tenant' : 'tenant_user';
 		}

package/mw/cache/model/mongo.js

@@ -22,10 +22,18 @@ let model = {
 		if (!model.mongo) {
 			model.mongo = new Mongo(dbConfiguration);
 			model.mongo.createIndex(cache_store, { 'l1': 1, 'l2': 1 }, { unique: true }, (err, index) => {
-				log.debug("Cache index: " + index + " created with error: " + err);
+				if (err) {
+					log.error("Cache index creation failed: " + err.message);
+				} else {
+					log.debug("Cache index created: " + index);
+				}
 			});
 			model.mongo.createIndex(cache_store, { 'expiresAt': 1 }, { expireAfterSeconds: 0 }, (err, index) => {
-				log.debug("Cache TTL index: " + index + " created with error: " + err);
+				if (err) {
+					log.error("Cache TTL index creation failed: " + err.message);
+				} else {
+					log.debug("Cache TTL index created: " + index);
+				}
 			});
 		}
 	},

package/mw/gotoService/index.js

@@ -50,7 +50,7 @@ module.exports = (configuration) => {
 			//create proxy info object before calling extractbuildparams
 			let proxy = false;
 			if (serviceInfo[1] === 'proxy' && serviceInfo[2] === 'redirect') {
-				req.soajs.log.error("Route: [/proxy/redirect] is deprecated. You should use [/soajs/proxy].");
+				req.soajs.log.info("Route: [/proxy/redirect] is deprecated. You should use [/soajs/proxy].");
 				proxy = true;
 			} else if (serviceInfo[1] === 'soajs' && serviceInfo[2] === 'proxy') {
 				proxy = true;

package/mw/gotoService/redirectToService.js

@@ -81,7 +81,8 @@ function flushMonitorBuffer() {
 				}
 			})
 			.catch((error) => {
-				req.soajs.log.debug('Monitor buffer flush failed: ' + error.message);
+				//NOTE: the buffer was already cleared, these items are lost
+				req.soajs.log.error('Monitor buffer flush failed, ' + itemsToSend.length + ' items lost: ' + error.message);
 			});
 	});
 }

package/mw/idempotency/index.js

@@ -11,6 +11,7 @@
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 const memory = require('./model/memory.js');
 const mongo = require('./model/mongo.js');
+const { logOnce } = require('../../lib/logOnce.js');
 
 let model = { memory, mongo };
 
@@ -119,7 +120,8 @@ module.exports = function (configuration) {
 
 		const modelType = idempotencyConfig.model || 'memory';
 		if (!model[modelType]) {
-			configuration.log.warn('Idempotency: Unknown model [' + modelType + ']. It can only be [memory || mongo]');
+			logOnce(configuration.log, "error", "idempotency_model_" + modelType,
+				'Idempotency: Unknown model [' + modelType + ']. It can only be [memory || mongo]. Idempotency is off.');
 			return next();
 		}
 

package/mw/idempotency/model/mongo.js

@@ -22,10 +22,18 @@ let model = {
 		if (!model.mongo) {
 			model.mongo = new Mongo(dbConfiguration);
 			model.mongo.createIndex(idempotency_store, { 'l1': 1, 'l2': 1 }, { unique: true }, (err, index) => {
-				log.debug("Idempotency index: " + index + " created with error: " + err);
+				if (err) {
+					log.error("Idempotency index creation failed: " + err.message);
+				} else {
+					log.debug("Idempotency index created: " + index);
+				}
 			});
 			model.mongo.createIndex(idempotency_store, { 'expiresAt': 1 }, { expireAfterSeconds: 0 }, (err, index) => {
-				log.debug("Idempotency TTL index: " + index + " created with error: " + err);
+				if (err) {
+					log.error("Idempotency TTL index creation failed: " + err.message);
+				} else {
+					log.debug("Idempotency TTL index created: " + index);
+				}
 			});
 		}
 	},

package/mw/keyACL/index.js

@@ -48,7 +48,7 @@ module.exports = () => {
 					if (obj.service_v) {
 						let san_v = coreLibs.version.sanitize(obj.service_v);
 						if (!aclObj[san_v]) {
-							obj.req.soajs.log.error("Problem accessing service [" + obj.req.soajs.controller.serviceParams.service_n + "] & version [" + obj.req.soajs.controller.serviceParams.service_v + "]");
+							//NOTE: the error context already reports the service and the version
 							return cb(154, null);
 						}
 						aclObj = aclObj[san_v];

package/mw/mt/index.js

@@ -48,7 +48,7 @@ module.exports = (configuration) => {
 				serviceInfo = req.soajs.controller.serviceParams.registry.versions[req.soajs.controller.serviceParams.version];
 			}
 			if (!serviceInfo) {
-				req.soajs.log.error("Problem accessing service [" + req.soajs.controller.serviceParams.name + "], API [" + req.soajs.controller.serviceParams.path + "] & version [" + req.soajs.controller.serviceParams.version + "]");
+				//NOTE: the error context already reports the service, the api and the version
 				return next(133);
 			}
 			let oauth = true;
@@ -160,12 +160,16 @@ module.exports = (configuration) => {
 								//if this is controller route: /key/permission/get, ignore async waterfall response
 								if (url_keyACL) {
 									if (!req.soajs.uracDriver) {
-										if (err && err.message) {
-											req.soajs.log.error(err.message);
-										}
 										if (err && err.name === "OAuth2Error") {
+											//NOTE: the code and the message travel with the error, the
+											//		error context reports both with full attribution
 											return next(err);
 										}
+										//NOTE: the cause is replaced by 158 below, so it is only
+										//		recoverable from here
+										if (err && err.message) {
+											req.soajs.log.error(err.message);
+										}
 										//doesn't work if you are not logged in
 										return next(158);
 									} else {
@@ -180,7 +184,8 @@ module.exports = (configuration) => {
 								}
 
 								if (err) {
-									req.soajs.log.error("Problem accessing service [" + req.soajs.controller.serviceParams.name + "], API [" + req.soajs.controller.serviceParams.path + "] & version [" + req.soajs.controller.serviceParams.version + "]");
+									//NOTE: the error context already reports the service, the api and
+									//		the version alongside the code and the message
 									return next(err);
 								} else {
 									let serviceName = data.req.soajs.controller.serviceParams.name;
@@ -330,7 +335,9 @@ module.exports = (configuration) => {
 															return callback();
 														});
 													} else {
-														req.soajs.log.debug(serviceName + " interConnect failed for [" + item.name + "@" + item.version + "]");
+														//NOTE: the peer is dropped from the injected address book, the calling
+														//		service will behave as if it was never configured
+														req.soajs.log.error(serviceName + " interConnect failed for [" + item.name + "@" + item.version + "], peer dropped from awareness");
 														return callback();
 													}
 												});

package/mw/mt/utils.js

@@ -190,8 +190,7 @@ let utils = {
 						let block = getCachedNetmask(addr);
 						return block && block.contains(ip);
 					} catch (err) {
-						obj.req.soajs.log.error('IP whitelist security configuration failed: ', addr);
-						obj.req.soajs.log.error(err);
+						obj.req.soajs.log.error('IP whitelist security configuration failed for [' + addr + ']: ' + (err.message || err));
 					}
 					return false;
 				}));
@@ -320,8 +319,7 @@ let utils = {
 					let block = getCachedNetmask(addr);
 					return block && block.contains(ip);
 				} catch (err) {
-					obj.req.soajs.log.error('Geographic security configuration failed: ', addr);
-					obj.req.soajs.log.error(err.message);
+					obj.req.soajs.log.error('Geographic security configuration failed for [' + addr + ']: ' + (err.message || err));
 				}
 				return false;
 			}));

package/mw/traffic/index.js

@@ -12,6 +12,7 @@
 const get = (p, o) => p.reduce((xs, x) => (xs && xs[x]) ? xs[x] : null, o);
 const memory = require('./model/memory.js');
 const mongo = require('./model/mongo.js');
+const { logOnce } = require('../../lib/logOnce.js');
 
 let model = {
 	memory, mongo
@@ -175,7 +176,8 @@ module.exports = function (configuration) {
 				if (trafficModel) {
 					throttling.model = trafficModel;
 					if (!model[throttling.model]) {
-						req.soajs.log.warn('Throttling:', 'Unkown model [' + throttling.model + ']. It can only be [memory || mongo]');
+						logOnce(req.soajs.log, "error", "throttling_model_" + throttling.model,
+							'Throttling: Unknown model [' + throttling.model + ']. It can only be [memory || mongo]. Throttling is off.');
 						return next();
 					}
 				}

package/mw/traffic/model/mongo.js

@@ -23,7 +23,11 @@ let model = {
         if (!model.mongo) {
             model.mongo = new Mongo(dbConfiguration);
             model.mongo.createIndex(throttling_monitor, { 'l1': 1, 'l2': 1 }, {}, (err, index) => {
-                log.debug("Index: " + index + " created with error: " + err);
+                if (err) {
+                    log.error("Throttling index creation failed: " + err.message);
+                } else {
+                    log.debug("Throttling index created: " + index);
+                }
             });
         }
     },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "soajs.controller",
   "description": "soajs multi tenant API gateway",
-  "version": "4.3.11",
+  "version": "4.3.13",
   "author": {
     "name": "soajs team",
     "email": "team@soajs.org"

package/server/controller.js

@@ -74,7 +74,7 @@ Controller.prototype.init = function (callback) {
 			let log = core.getLogger(_self.soajs.param.init.serviceName, registry.serviceConfig.logger);
 			if (service.fetched) {
 				if (!service.fetched.result) {
-					log.warn("Unable to find the service host ip. The service will NOT be registered for awareness.");
+					log.error("Unable to find the service host ip. The service will NOT be registered for awareness.");
 					log.info("IPs found: ", service.fetched.extra.ips);
 					log.warn("The default service IP has been used [" + service.ip + "]");
 				} else {
@@ -316,7 +316,7 @@ Controller.prototype.start = function (registry, log, service, server, serverMai
 								});
 							});
 						} else {
-							log.warn("Unable to register host IP [" + service.ip + "] for service [" + _self.soajs.param.init.serviceName + "@" + _self.soajs.param.init.serviceVersion + "]");
+							log.error("Unable to register host IP [" + service.ip + "] for service [" + _self.soajs.param.init.serviceName + "@" + _self.soajs.param.init.serviceVersion + "]");
 						}
 					});
 				}

package/server/maintenance.js

@@ -145,7 +145,7 @@ let reloadRegistry = (parsedUrl, core, log, param, serviceIp, cb) => {
 	}, function (err, reg) {
 		let response = maintenanceResponse(parsedUrl, param);
 		if (err) {
-			log.warn("Failed to load registry. reusing from previous load. Reason: " + err.message);
+			log.error("Failed to load registry. reusing from previous load. Reason: " + err.message);
 		} else {
 			response.result = true;
 			response.data = cloneAndFilterRegistry(reg);
@@ -236,7 +236,7 @@ let Maintenance = (core, log, param, serviceIp, regEnvironment, awareness_mw, so
 							infoObj = body;
 						}
 						if (!soajsLib.version.validate(infoObj.version)) {
-							log.warn("Failed to register service for [" + infoObj.name + "@" + infoObj.version + "] version should be of format [1.1]");
+							log.error("Failed to register service for [" + infoObj.name + "@" + infoObj.version + "] version should be of format [1.1]");
 							// res.writeHead(200, {'Content-Type': 'application/json'});
 							return res.end(JSON.stringify(response));
 						}
@@ -287,7 +287,7 @@ let Maintenance = (core, log, param, serviceIp, regEnvironment, awareness_mw, so
 									response.result = true;
 									response.data = data;
 								} else {
-									log.warn("Failed to register service for [" + infoObj.name + "] " + err.message);
+									log.error("Failed to register service for [" + infoObj.name + "] " + err.message);
 								}
 								return res.end(JSON.stringify(response));
 							});

package/test/unit/index.js

@@ -10,6 +10,7 @@ describe("Starting Gateway Unit test", () => {
         require("./lib/apiPathParam2apiRegExp.js");
         require("./lib/http.js");
         require("./lib/ip.js");
+        require("./lib/logOnce.js");
         require("./lib/param.js");
         require("./lib/parseURL.js");
 

package/test/unit/lib/logOnce.js

@@ -0,0 +1,81 @@
+"use strict";
+
+/**
+ * @license
+ * Copyright SOAJS All Rights Reserved.
+ *
+ * Use of this source code is governed by an Apache license that can be
+ * found in the LICENSE file at the root of this repository
+ */
+
+const assert = require('assert');
+const helper = require("../../helper.js");
+
+let logOnceLib = helper.requireModule('./lib/logOnce.js');
+const { logOnce, resetLogOnce } = logOnceLib;
+
+describe("Unit test for: lib - logOnce", () => {
+
+	let emitted = [];
+	let log = {
+		error: (input) => {
+			emitted.push({ "level": "error", "input": input });
+		},
+		warn: (input) => {
+			emitted.push({ "level": "warn", "input": input });
+		}
+	};
+
+	beforeEach(() => {
+		emitted = [];
+		resetLogOnce();
+	});
+
+	it("emits the first time", (done) => {
+		let did = logOnce(log, "error", "k1", "boom");
+		assert.strictEqual(did, true);
+		assert.strictEqual(emitted.length, 1);
+		assert.strictEqual(emitted[0].level, "error");
+		assert.strictEqual(emitted[0].input, "boom");
+		done();
+	});
+
+	it("drops repeats of the same key", (done) => {
+		for (let i = 0; i < 100; i++) {
+			logOnce(log, "error", "k1", "boom");
+		}
+		assert.strictEqual(emitted.length, 1);
+		done();
+	});
+
+	it("keeps distinct keys separate", (done) => {
+		logOnce(log, "error", "k1", "one");
+		logOnce(log, "error", "k2", "two");
+		logOnce(log, "error", "k1", "one again");
+		assert.strictEqual(emitted.length, 2);
+		assert.strictEqual(emitted[0].input, "one");
+		assert.strictEqual(emitted[1].input, "two");
+		done();
+	});
+
+	it("honours the requested level", (done) => {
+		logOnce(log, "warn", "k3", "careful");
+		assert.strictEqual(emitted[0].level, "warn");
+		done();
+	});
+
+	it("does not throw on a missing logger or unknown level", (done) => {
+		assert.strictEqual(logOnce(null, "error", "k4", "x"), false);
+		assert.strictEqual(logOnce(log, "nosuchlevel", "k5", "x"), false);
+		assert.strictEqual(emitted.length, 0);
+		done();
+	});
+
+	it("does not consume the key when it could not emit", (done) => {
+		logOnce(null, "error", "k6", "x");
+		let did = logOnce(log, "error", "k6", "now it works");
+		assert.strictEqual(did, true);
+		assert.strictEqual(emitted.length, 1);
+		done();
+	});
+});

package/test/unit/utilities/utils.js

@@ -109,6 +109,56 @@ describe("Testing utilities", () => {
 		});
 	});
 
+	it("logErrors - no longer duplicates what the error context reports", (done) => {
+		// the full chain: logErrors -> controllerClientErrorHandler -> controllerErrorHandler
+		let r = makeReq(true);
+		let oauthErr = new Error("The access token provided is invalid.");
+		oauthErr.name = "OAuth2Error";
+		oauthErr.code = 401;
+
+		utils.logErrors(oauthErr, r, res, (e1) => {
+			utils.controllerClientErrorHandler(e1, r, res, (e2) => {
+				utils.controllerErrorHandler(e2, r, res, null);
+			});
+		});
+
+		assert.strictEqual(logged.error.length, 1, "expected exactly one error line for one failure");
+		let ctx = JSON.parse(logged.error[0]);
+		assert.strictEqual(ctx.code, 401);
+		assert.strictEqual(ctx.msg, "The access token provided is invalid.");
+		done();
+	});
+
+	it("logErrors - keeps the underlying cause when it becomes a generic 164", (done) => {
+		// an error with neither code nor msg is replaced by 164, so the context line
+		// cannot report the original. logErrors must still record it.
+		let r = makeReq(true);
+		utils.logErrors({ "message": "socket hang up" }, r, res, (e1) => {
+			utils.controllerClientErrorHandler(e1, r, res, (e2) => {
+				utils.controllerErrorHandler(e2, r, res, null);
+			});
+		});
+
+		assert.strictEqual(logged.error.length, 2);
+		assert.strictEqual(logged.error[0], "socket hang up", "underlying cause must survive");
+		let ctx = JSON.parse(logged.error[1]);
+		assert.strictEqual(ctx.code, 164);
+		done();
+	});
+
+	it("errorContext - falls back to service_n and service_v before gotoService resolves them", (done) => {
+		// key and keyACL run before gotoService, so serviceParams carries only the
+		// raw service_n / service_v at that point
+		let r = makeReq(true);
+		r.soajs.controller.serviceParams = { "service_n": "urac", "service_v": "3" };
+		utils.controllerErrorHandler(154, r, res, null);
+
+		let ctx = JSON.parse(logged.error[0]);
+		assert.strictEqual(ctx.service, "urac");
+		assert.strictEqual(ctx.version, "3");
+		done();
+	});
+
 	it("logErrors - string error", (done) => {
 		utils.logErrors("error", req, res, (error) => {
 			done();

package/utilities/utils.js

@@ -51,8 +51,10 @@ function errorContext(err, req) {
         "code": code,
         "msg": msg,
         "method": req.method,
+        //NOTE: name and version are resolved by gotoService, the service_n and service_v
+        //		fallbacks cover the middleware that runs before it, ex: key and keyACL
         "service": serviceParams.name || serviceParams.service_n || null,
-        "version": serviceParams.version || null,
+        "version": serviceParams.version || serviceParams.service_v || null,
         "api": (serviceParams.parsedUrl && serviceParams.parsedUrl.pathname) || null,
         "url": redactUrl(req.url),
         "ip": req.getClientIP ? req.getClientIP() : null,
@@ -86,13 +88,15 @@ function errorContext(err, req) {
  * @param next
  */
 function logErrors(err, req, res, next) {
+    //NOTE: every error routed here reaches controllerErrorHandler, which logs a
+    //      structured context carrying the code and the message. The cases below
+    //      only log what that context cannot reproduce, which is the underlying
+    //      cause of an error that gets replaced by the generic 164.
     if (typeof err === "number") {
-        req.soajs.log.error(core.error.generate(err).message);
         return next(err);
     }
     if (typeof err === "object") {
         if (err.code && err.message) {
-            req.soajs.log.error(err.message);
             if (err.name === "OAuth2Error") {
                 return next({ "code": err.code, "status": err.code, "msg": err.message });
             } else {
@@ -100,15 +104,12 @@ function logErrors(err, req, res, next) {
             }
         } else if (err.code && err.msg) {
             err.message = err.msg;
-            req.soajs.log.error(err.message);
             return next(err);
         } else {
             req.soajs.log.error(err.message || err);
-            req.soajs.log.error(core.error.generate(164).message);
         }
     } else {
         req.soajs.log.error(err);
-        req.soajs.log.error(core.error.generate(164).message);
     }
 
     return next(core.error.getError(164));