whatap 0.4.89 → 0.4.91

package/lib/conf/config-default.js

@@ -137,6 +137,8 @@ var ConfigDefault = {
     "status_ignore_set": str('status_ignore_set', ''),
     "httpc_status_ignore_set": str('httpc_status_ignore_set', ''),
     "trace_sql_normalize_enabled" : true,
+    "httpc_not_found_ignore": bool('httpc_not_found_ignore', false),
+    "httpc_not_found_ignore_time": num('httpc_not_found_ignore', 300000),
 
     //2017.05.02 AUTO ONAME
     "auto_oname_enabled" : bool('auto_oname_enabled',false),

package/lib/observers/http-observer.js

@@ -35,7 +35,8 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     PluginLoaderManager = require('../plugin/plugin-loadermanager'),
     Long                = require('long'),
     KeyGen              = require('../util/keygen'),
-    Logger              = require('../logger');
+    Logger              = require('../logger'),
+    IntKeyLinkedMap = require("../util/intkey-linkedmap");
 
 var _exts=new Set([".css",".js",".png", ".htm", ".html", ".gif", ".jpg", ".css", ".txt", ".ico"]);
 
@@ -50,6 +51,8 @@ var httpc_status_ignore = conf.getProperty('httpc_status_ignore', '');
 var status_ignore_set = conf.getProperty('status_ignore_set', '');
 var httpc_status_ignore_set = conf.getProperty('httpc_status_ignore_set', '');
 var profile_error_step_enabled = conf.getProperty('profile_error_step_enabled', '');
+var httpc_not_found_ignore = conf.getProperty('httpc_not_found_ignore', false);
+var httpc_not_found_ignore_time = conf.getProperty('httpc_not_found_ignore_time', 300000);
 conf.on('trace_http_client_ip_header_key', function(newProperty) {
     configIpHeaderKey = newProperty;
 });
@@ -92,6 +95,12 @@ conf.on('httpc_status_ignore_set', function (newProps) {
 conf.on('profile_error_step_enabled', function (newProps) {
     profile_error_step_enabled = newProps;
 })
+conf.on('httpc_not_found_ignore', function (newProps) {
+    httpc_not_found_ignore = newProps;
+})
+conf.on('httpc_not_found_ignore_time', function (newProps) {
+    httpc_not_found_ignore_time = newProps;
+})
 var staticConents = function (newProps) {
     var x=new Set();
     var words = !newProps?[]:newProps.split(',');
@@ -111,6 +120,9 @@ var staticConents = function (newProps) {
 conf.on('web_static_content_extensions', staticConents);
 staticConents(conf["web_static_content_extensions"]);
 
+var httpc_not_found_ignore_map = null;
+var httpc_not_found_ignore_url = [];
+
 var HttpObserver = function(agent){
     this.agent = agent;
     this.packages = ['http','https'];
@@ -144,6 +156,17 @@ HttpObserver.prototype.__createTransactionObserver = function(callback, isHttps,
                 if(ctx == null) { return; }
                 PluginLoaderManager.do('httpserviceend', ctx, req, res);
 
+                var not_found_ignore = httpc_not_found_ignore_url.includes(req.url);
+                if (httpc_not_found_ignore && !not_found_ignore && obj.statusCode === 404) {
+                    const url_hash = HashUtil.hashFromString(req.url);
+                    const currentTime = new Date().getTime();
+
+                    if (!httpc_not_found_ignore_map) {
+                        httpc_not_found_ignore_map = new IntKeyLinkedMap(500, 1).setMax(500);
+                    }
+                    updateNotFoundIgnoreMap(req.url, url_hash, currentTime);
+                }
+
                 ctx.isStaticContents = isStatic(req.url);
                 ctx.http_method = req.method;
                 if(conf.profile_http_querystring_enabled){
@@ -163,16 +186,11 @@ HttpObserver.prototype.__createTransactionObserver = function(callback, isHttps,
                 var requestPath = '';
                 if(req.route && req.route.path)
                     requestPath = req.route.path;
-                var is_ignore;
-                if(ctx.is_httpc_error){
-                    is_ignore = shouldIgnoreError(res.statusCode, requestPath, httpc_status_ignore, httpc_status_ignore_set);
-                } else {
-                    is_ignore = shouldIgnoreError(res.statusCode, requestPath, status_ignore, status_ignore_set);
-                }
 
-                if(!is_ignore)
+                var shouldEndTransaction = shouldEndCurrentTransaction(not_found_ignore, ctx, res, req.route ? req.route.path : '');
+                if (shouldEndTransaction) {
                     self.__endTransaction(null, ctx, req, res);
-                else{
+                } else {
                     TraceContextManager.end(ctx._id);
                     ctx = null;
                 }
@@ -792,4 +810,33 @@ function shouldIgnoreError(statusCode, url, statusCodeIgnore, statusIgnoreSet) {
     return false;
 }
 
+function updateNotFoundIgnoreMap(url, url_hash, currentTime) {
+    var ignore_value = httpc_not_found_ignore_map.get(url_hash) || {
+        count: 0, start_time: currentTime, last_time: currentTime, url: url
+    };
+
+    ignore_value.count++;
+    ignore_value.last_time = currentTime;
+
+    if ((ignore_value.last_time - ignore_value.start_time) > httpc_not_found_ignore_time) {
+        ignore_value = { count: 1, start_time: currentTime, last_time: currentTime, url: url };
+    }
+
+    if (ignore_value.count >= 50) {
+        httpc_not_found_ignore_url.push(url);
+        httpc_not_found_ignore_map.remove(url_hash);
+        Logger.print('WHATAP-610', "The ignore option for URL " + url + " is applied.", false);
+    }
+
+    httpc_not_found_ignore_map.put(url_hash, ignore_value);
+}
+
+function shouldEndCurrentTransaction(not_found_ignore, ctx, res, requestPath) {
+    if (not_found_ignore) return false;
+    var is_ignore = ctx.is_httpc_error ?
+        shouldIgnoreError(res.statusCode, requestPath, httpc_status_ignore, httpc_status_ignore_set) :
+        shouldIgnoreError(res.statusCode, requestPath, status_ignore, status_ignore_set);
+    return !is_ignore;
+}
+
 exports.HttpObserver = HttpObserver;

package/lib/observers/mongoose-observer.js

@@ -3,81 +3,157 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     SqlStepX            = require('../step/sql-stepx'),
     DataTextAgent       = require('../data/datatext-agent'),
     HashUtil            = require('../util/hashutil');
+const DBCStep = require("whatap/lib/step/dbc-step");
+const Logger = require("whatap/lib/logger");
 
 var MongooseObserver = function (agent) {
     this.agent = agent;
     this.packages = ['mongoose'];
 };
 
+var dbc_step, dbc, conn_dbc_hash;
 MongooseObserver.prototype.inject = function (mod, modName) {
 
     var self = this;
-
     var hookCommand = [
-        'find',
-        'remove',
-        'findById',
-        'findOne',
-        'count',
-        'distinct',
-        'findOneAndUpdate',
-        'findByIdAndUpdate',
-        'findOneAndRemove',
-        'findByIdAndRemove',
-        'create',
-        'insertMany',
-        'update',
-        'updateMany',
-        'updateOne',
-        'mapReduce',
-        'geoNear',
-        'geoSearch',
-        'populate'
+        "create",             // Create: Create a new document
+        "insertMany",         // Create: Create multiple documents at once
+        "find",               // Read: Find documents that match criteria
+        "findById",           // Read: Find a document by its ID
+        "findOne",            // Read: Find the first document that matches criteria
+        "countDocuments",     // Read: Count the number of documents that match criteria
+        "distinct",           // Read: Find distinct values for a given field
+        "updateMany",         // Update: Update multiple documents that match criteria
+        "updateOne",          // Update: Update a single document that matches criteria
+        "replaceOne",         // Update: Replace one document with another
+        "findOneAndUpdate",   // Update: Find the first document that matches criteria and update it
+        "findByIdAndUpdate",  // Update: Find a document by its ID and update it
+        "deleteOne",          // Delete: Delete a single document that matches criteria
+        "deleteMany",         // Delete: Delete multiple documents that match criteria
+        "findOneAndDelete",   // Delete: Find the first document that matches criteria and delete it
+        "findByIdAndDelete"   // Delete: Find a document by its ID and delete it
     ];
 
-    self.agent.aop.before(mod.Model, 'aggregate', function (obj, args) {
-        var ctx = TraceContextManager.getCurrentContext();
-        if(ctx == null) {return;}
+    self.agent.aop.after(mod, ['connect'] ,function (obj, args, ret) {
+        if(!args[0]) { return; }
+        if(dbc) { return; }
+        dbc = args[0];
+        conn_dbc_hash = HashUtil.hashFromString(dbc);
+
+        ret.then(data => {
+            self.agent.aop.both(data.Model, 'aggregate', function (_obj, _args, _lctx) {
+                var ctx = TraceContextManager.getCurrentContext();
+                if(!ctx) {return;}
+
+                if(dbc && conn_dbc_hash){
+                    DataTextAgent.DBC.add(conn_dbc_hash, dbc);
+                    DataTextAgent.METHOD.add(conn_dbc_hash, dbc);
+                    DataTextAgent.ERROR.add(conn_dbc_hash, dbc);
+                }
+
+                var dbc_step = new DBCStep();
+                dbc_step.hash = conn_dbc_hash;
+                dbc_step.start_time = ctx.getElapsedTime();
+
+                _lctx.dbc_step = dbc_step;
+
+                var sql_step = new SqlStepX();
+                sql_step.start_time = ctx.getElapsedTime();
+                _lctx.sql_step = sql_step;
+            }, function (_obj, _args, _ret, _lctx) {
+                var ctx = _lctx.context;
+                var dbc_step = _lctx.dbc_step;
+                var sql_step = _lctx.sql_step;
+                if(!ctx || !dbc_step || !sql_step) {return null;}
+
+                dbc_step.elapsed = ctx.getElapsedTime() - dbc_step.start_time;
+                ctx.profile.push(dbc_step);
+
+                ctx.footprint('Mongodb Command Start: ' + _ret.op );
 
-        var param = "";
-        if(Array.isArray(args[0])) {
-            args[0].forEach(function (val, i) {
-                if(i > 0 && param.length > 0) {
-                    param += ",";
+                const mongooseCollection = _ret._model;
+                try {
+                    var sql = mongooseCollection.modelName + ' aggregate';
+                    var param = "";
+                    if(Array.isArray(_args[0])) {
+                        _args[0].forEach(function (val, i) {
+                            if(i > 0 && param.length > 0) {
+                                param += ",";
+                            }
+                            if(val.hasOwnProperty('$match')) {
+                                var keys = Object.keys(val['$match']);
+                                param += keys;
+                            }
+
+                            if(val.hasOwnProperty('$group')) {
+                                var keys = Object.keys(val['$group']);
+                                param += keys;
+                            }
+                        })
+                    }
+                    sql += ' field=['+param+']';
+
+                    sql_step.hash = HashUtil.hashFromString(sql);
+                    sql_step.start_time = ctx.getElapsedTime();
+                    sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
+                    DataTextAgent.SQL.add(sql_step.hash, sql);
+                    ctx.profile.push(sql_step);
+                } catch(e) {
+                    Logger.printError("WHATAP-611", "Mongodb query error", e, false);
+                    sql_step = null;
                 }
-                if(val.hasOwnProperty('$match')) {
-                    var keys = Object.keys(val['$match']);
-                    param += keys;
+            });
+
+            self.agent.aop.both(data.Model, hookCommand, function (_obj, _args, _lctx) {
+                if(_args[0] && typeof _args[0] !== "object") { return; }
+                const ctx = TraceContextManager.getCurrentContext();
+                if(!ctx) { return; }
+                if(dbc && conn_dbc_hash){
+                    DataTextAgent.DBC.add(conn_dbc_hash, dbc);
+                    DataTextAgent.METHOD.add(conn_dbc_hash, dbc);
+                    DataTextAgent.ERROR.add(conn_dbc_hash, dbc);
                 }
 
-                if(val.hasOwnProperty('$group')) {
-                    var keys = Object.keys(val['$group']);
-                    param += keys;
+                var dbc_step = new DBCStep();
+                dbc_step.hash = conn_dbc_hash;
+                dbc_step.start_time = ctx.getElapsedTime();
+
+                _lctx.dbc_step = dbc_step;
+
+                var sql_step = new SqlStepX();
+                sql_step.start_time = ctx.getElapsedTime();
+                _lctx.sql_step = sql_step;
+            }, function (_obj, _args, _ret, _lctx) {
+                var ctx = _lctx.context;
+                var dbc_step = _lctx.dbc_step;
+                var sql_step = _lctx.sql_step;
+                if(!ctx || !dbc_step || !sql_step) {return null;}
+
+                dbc_step.elapsed = ctx.getElapsedTime() - dbc_step.start_time;
+                ctx.profile.push(dbc_step);
+
+                ctx.footprint('Mongodb Command Start: ' + _ret.op );
+
+                const mongooseCollection = _ret.mongooseCollection;
+                try {
+                    var sql = mongooseCollection.modelName + ' ' + _ret.op ;//+ ': ' + JSON.stringify(Object.keys(args[0]));
+                    if(_args[0]) { sql += ' field=' + JSON.stringify(Object.keys(_args[0])); }
+                    if(_args[1]) { sql += ' value=' + JSON.stringify(Object.keys(_args[1])); }
+
+                    sql_step.hash = HashUtil.hashFromString(sql);
+                    sql_step.start_time = ctx.getElapsedTime();
+                    sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
+                    DataTextAgent.SQL.add(sql_step.hash, sql);
+                    ctx.profile.push(sql_step);
+                } catch(e) {
+                    Logger.printError("WHATAP-611", "Mongodb query error", e, false);
+                    sql_step = null;
                 }
             })
-        }
-
-        var sql_step = new SqlStepX();
-        var sql = 'mongodb aggregate (' + param + ')';
-        sql_step.hash = HashUtil.hashFromString(sql);
-        sql_step.start_time = ctx.getElapsedTime();
-        sql_step.dbc = 0;
-        
-        DataTextAgent.SQL.add(sql_step.hash, sql);
-        ctx.profile.push(sql_step);
-
-        self.agent.aop.functionHook(args, -1, function () {
-            TraceContextManager.resume(ctx);
-            sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
-        });
-    });
-
-    self.agent.aop.before(mod.Model, hookCommand, function (obj, args) {
-        var ctx_id = TraceContextManager.getCurrentId();
-        self.agent.aop.functionHook(args, -1, function () {
-            TraceContextManager.resume(ctx_id);
-        });
-    });
+        }).catch(e => {
+            Logger.printError("WHATAP-612", "Mongodb connection error", e, false);
+        })
+    })
 
 };
 

package/lib/observers/net-observer.js

@@ -10,7 +10,6 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     IPUtil = require('../util/iputil'),
     Logger = require('../logger');
 
-
 var NetObserver = function (agent) {
     this.agent = agent;
     this.packages = ['net', 'dgram'];

package/lib/observers/socket.io-observer.js

@@ -5,51 +5,136 @@
  */
 
 var MeterSocketio = require('../counter/meter/meter-socket.io'),
-Logger              = require('../logger');
+    TraceContextManager = require('../trace/trace-context-manager'),
+    SocketStep = require('../step/socket-step'),
+    conf = require('../conf/configure'),
+    IPUtil = require('../util/iputil'),
+    Logger = require('../logger');
+const {Detector: URLPatternDetector} = require("whatap/lib/trace/serviceurl-pattern-detector");
+const HashUtil = require("whatap/lib/util/hashutil");
+const DataTextAgent = require("whatap/lib/data/datatext-agent");
+const ResourceProfile = require("whatap/lib/util/resourceprofile");
+const ProfilePack         = require('../pack/profile-pack');
+const TxRecord            = require('../service/tx-record');
+const DateUtil            = require('../util/dateutil');
+const SecurityMaster      = require('../net/security-master');
+const DataProfileAgent    = require('../data/dataprofile-agent');
+const MeterService        = require('../counter/meter/meter-service').MeterService;
 
 var SocketIOObserver = function(agent){
     this.agent = agent;
     this.packages = ['socket.io'];
 };
 
-SocketIOObserver.prototype.inject = function( mod ){
+SocketIOObserver.prototype.inject = function (mod, moduleName) {
+    if (mod.__whatap_observe__) {
+        return;
+    }
+    mod.__whatap_observe__ = true;
+    Logger.initPrint("SocketIOObserver");
+    if (conf.trace_background_socket_enabled === false) {
+        return;
+    }
+
     var self = this;
     var aop = self.agent.aop;
 
-    if(mod.__whatap_observe__) { return; }
-    mod.__whatap_observe__ = true;
-    Logger.initPrint("SocketIOObserver");
-    aop.after(mod.prototype, ['listen','attach'], function(obj, args, ret) {
-        if (!ret.sockets) return;
+    aop.before(mod.Server?.prototype, 'on', function (obj, args, ret, lctx) {
+        args[args.length-1] = self.__createTransactionObserver(args[args.length-1]);
+    })
+};
+
+SocketIOObserver.prototype.__createTransactionObserver = function (callback) {
+    var self = this;
+
+    return function (){
+        TraceContextManager._asyncLocalStorage.run(initCtx(), () => {
+            var ctx = TraceContextManager._asyncLocalStorage.getStore();
+            if(!ctx) {
+                return;
+            }
+
+            MeterSocketio.connected(1);
+
+            var host;
+            const socket = arguments[0];
+            const address = socket.handshake.address;
+            if(address && address.includes(':')){
+                host = address.substring(address.lastIndexOf(':')+1);
+            }
+
+            ctx.socket_connecting = true;
+
+            ctx.footprint('Socket Connecting: ' + host);
 
-        aop.before(ret.sockets, ['on', 'addListener'], function(obj, args) {
-            if (args[0] !== 'connection') return;
+            var step = new SocketStep();
+            step.start_time = ctx.getElapsedTime();
+            step.ipaddr = Buffer.from(IPUtil.stringToBytes(host));
+            // step.port = port;
 
-            aop.functionHook(args, -1, function(obj, args) {
-                if (!args[0]) return; //no client socket
+            ctx.socket_connecting = false;
+            step.elapsed = ctx.getElapsedTime() - step.start_time;
+            ctx.profile.push(step);
 
-                var socket = args[0];
+            ctx.footprint('Socket Connecting Done');
+
+            self.__endTransaction(null, ctx);
+
+            return;
+        })
+    }
+}
+
+SocketIOObserver.prototype.__endTransaction = function(error, ctx) {
+    try {
+        var profile = new ProfilePack();
+        var wtx = new TxRecord();
+        wtx.endTime = DateUtil.currentTime();
+        profile.time  = wtx.endTime;
+        wtx.elapsed = ctx.getElapsedTime();
+
+        ctx.service_hash = HashUtil.hashFromString(ctx.service_name);
+        DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+        wtx.service = ctx.service_hash;
+        wtx.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
+        wtx.malloc = ResourceProfile.getUsedHeapSize()-ctx.start_malloc;
+        if(wtx.malloc < 0) { wtx.malloc = 0; }
+        wtx.status = 2;
+
+        MeterService.add(wtx.service, wtx.elapsed,
+            wtx.errorLevel, ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+
+        profile.oid = SecurityMaster.OID;
+        profile.service = wtx;
+
+        TraceContextManager.end(ctx._id);
+        setTimeout(function () {
+            DataProfileAgent.sendProfile(ctx, profile, false);
+            ctx = null;
+        }, 100);
+
+    } catch (e) {
+        Logger.printError('WHATAP-607', 'End transaction error..', e, false);
+        TraceContextManager.end(ctx._id);
+        ctx = null;
+    }
+
+};
 
-                MeterSocketio.connected(1);
+function initCtx() {
+    const ctx = TraceContextManager.start();
+    if (!ctx) {
+        return;
+    }
 
-                socket.on('disconnect', function() {
-                    MeterSocketio.disconnected();
-                });
+    ctx.start_malloc = ResourceProfile.getUsedHeapSize();
+    ctx.start_cpu = ResourceProfile.getCPUTime();
 
-                aop.before(socket, ['emit', 'send'], function(obj, args) {
-                    if (args[0] === 'newListener') return;
-                    MeterSocketio.send(null, args);
-                });
+    var remote_addr = IPUtil.checkIp4("0.0.0.0");
+    ctx.remoteIp = IPUtil.stringToInt(remote_addr);
 
-                aop.before(socket, ['on', 'addListener'], function(obj, args) {
-                    if (args[0] === 'disconnect') return;
-                    aop.functionHook(args, -1, function (obj, args) {
-                        MeterSocketio.receive(null, args[0]);
-                    });
-                });
-            });
-        });
-    });
+    return ctx;
 }
 
 exports.SocketIOObserver = SocketIOObserver;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "whatap",
   "homepage": "http://www.whatap.io",
-  "version": "0.4.89",
-  "releaseDate": "20240122",
+  "version": "0.4.91",
+  "releaseDate": "20240205",
   "description": "Monitoring and Profiling Service",
   "main": "index.js",
   "scripts": {},