whatap 0.4.93 → 0.4.94

package/lib/conf/config-default.js

@@ -57,7 +57,9 @@ var ConfigDefault = {
 
     "profile_http_querystring_enabled": bool('profile_http_querystring_enabled',false),
     "profile_http_header_enabled": bool('profile_http_header_enabled',false),
+    "profile_http_header_ignore_keys": str('profile_http_header_ignore_keys','Cookie,cookie,accept,user-agent,referer'),
     "profile_http_parameter_enabled": bool('profile_http_parameter_enabled',false),
+    "profile_http_parameter_keys": str('profile_http_parameter_keys',''),
     "profile_httpc_parameter_enabled": bool('profile_httpc_parameter_enabled',false),
     "profile_http_parameter_url_prefix": "/",
     "profile_http_header_url_prefix": "/",
@@ -85,6 +87,7 @@ var ConfigDefault = {
     "trace_user_agent_header_key": str('trace_user_agent_header_key',''),
     "trace_referer_header_key": str('trace_referer_header_key',''),
     "profile_sql_param_enabled": bool('profile_sql_param_enabled',false),
+    "profile_sql_resource_enabled": bool('profile_sql_resource_enabled',false),
     "error_snap_enabled" : true,
     "trace_user_enabled": bool('trace_user_enabled',true),
     "trace_user_using_ip": bool('trace_user_using_ip',true),
@@ -96,6 +99,9 @@ var ConfigDefault = {
     "trace_sql_error_depth": num('trace_sql_error_depth', 50),
     "transaction_status_error_enable": bool("transaction_status_error_enable", true),
 
+    "trace_sampling_enabled": bool("trace_sampling_enabled", false),
+    "trace_sampling_tps": bool("trace_sampling_tps", 1000),
+
     "profile_error_sql_time_max": num("profile_error_sql_time_max", 30000),
     "profile_error_step_enabled": bool('profile_error_step_enabled', true),
     "hook_direct_patch_classes": "",
@@ -139,6 +145,7 @@ var ConfigDefault = {
     "trace_sql_normalize_enabled" : true,
     "httpc_not_found_ignore": bool('httpc_not_found_ignore', false),
     "httpc_not_found_ignore_time": num('httpc_not_found_ignore', 300000),
+    "ignore_sql_error_code_set": str('ignore_sql_error_code_set', ''),
 
     //2017.05.02 AUTO ONAME
     "auto_oname_enabled" : bool('auto_oname_enabled',false),

package/lib/observers/http-observer.js

@@ -32,11 +32,14 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     ResourceProfile     = require('../util/resourceprofile'),
     TextTypes           = require('../lang/text-types'),
     MessageStep         = require('../step/message-step'),
+    SecureMsgStep = require('../step/securemsg-step'),
     PluginLoaderManager = require('../plugin/plugin-loadermanager'),
     Long                = require('long'),
     KeyGen              = require('../util/keygen'),
     Logger              = require('../logger'),
     IntKeyLinkedMap = require("../util/intkey-linkedmap");
+const ParamSecurity = require("../util/paramsecurity");
+const {Buffer} = require("buffer");
 
 var _exts=new Set([".css",".js",".png", ".htm", ".html", ".gif", ".jpg", ".css", ".txt", ".ico"]);
 
@@ -53,6 +56,9 @@ var httpc_status_ignore_set = conf.getProperty('httpc_status_ignore_set', '');
 var profile_error_step_enabled = conf.getProperty('profile_error_step_enabled', '');
 var httpc_not_found_ignore = conf.getProperty('httpc_not_found_ignore', false);
 var httpc_not_found_ignore_time = conf.getProperty('httpc_not_found_ignore_time', 300000);
+var profile_http_header_ignore_keys = conf.getProperty('profile_http_header_ignore_keys', 'Cookie,cookie,accept,user-agent,referer');
+var profile_http_parameter_enabled = conf.getProperty('profile_http_parameter_enabled', true);
+var profile_http_parameter_keys = conf.getProperty('profile_http_parameter_keys', '');
 conf.on('trace_http_client_ip_header_key', function(newProperty) {
     configIpHeaderKey = newProperty;
 });
@@ -101,6 +107,15 @@ conf.on('httpc_not_found_ignore', function (newProps) {
 conf.on('httpc_not_found_ignore_time', function (newProps) {
     httpc_not_found_ignore_time = newProps;
 })
+conf.on('profile_http_header_ignore_keys', function (newProps) {
+    profile_http_header_ignore_keys = newProps;
+})
+conf.on('profile_http_parameter_enabled', function (newProps) {
+    profile_http_parameter_enabled = newProps;
+})
+conf.on('profile_http_parameter_keys', function (newProps) {
+    profile_http_parameter_keys = newProps;
+})
 var staticConents = function (newProps) {
     var x=new Set();
     var words = !newProps?[]:newProps.split(',');
@@ -121,7 +136,7 @@ conf.on('web_static_content_extensions', staticConents);
 staticConents(conf["web_static_content_extensions"]);
 
 var httpc_not_found_ignore_map = null;
-var httpc_not_found_ignore_url = [];
+var httpc_not_found_ignore_url = new Set();
 
 var HttpObserver = function(agent){
     this.agent = agent;
@@ -156,7 +171,7 @@ HttpObserver.prototype.__createTransactionObserver = function(callback, isHttps,
                 if(ctx == null) { return; }
                 PluginLoaderManager.do('httpserviceend', ctx, req, res);
 
-                var not_found_ignore = httpc_not_found_ignore_url.includes(req.url);
+                var not_found_ignore = httpc_not_found_ignore_url.has(req.url);
                 if (httpc_not_found_ignore && !not_found_ignore && obj.statusCode === 404) {
                     const url_hash = HashUtil.hashFromString(req.url);
                     const currentTime = new Date().getTime();
@@ -324,10 +339,28 @@ function initCtx(req, res) {
         var step = new MessageStep();
         step.hash = HashUtil.hashFromString("HTTP-HEADERS");
         step.start_time = ctx.getElapsedTime();
-        if(req.headers && Object.keys(req.headers).length > 0)
-            step.desc = Object.keys(req.headers).map(key => `${key}=${req.headers[key]}`).join('\n');
-        DataTextAgent.MESSAGE.add(step.hash, "HTTP-HEADERS");
-        ctx.profile.push(step);
+
+        var header_ignore_key_set = new Set();
+        if(profile_http_header_ignore_keys) {
+            header_ignore_key_set = new Set(profile_http_header_ignore_keys.split(','));
+        }
+
+        try{
+            step.desc = Object.keys(req.headers).map(function (key) {
+                if (!header_ignore_key_set.has(key)) {
+                    return `${key}=${req.headers[key]}`;
+                }
+            }).filter(element => {
+                if (element) return element
+            }).join('\n');
+
+            DataTextAgent.MESSAGE.add(step.hash, "HTTP-HEADERS");
+            ctx.profile.push(step);
+        }catch (e) {
+            step = null;
+            header_ignore_key_set = null;
+            Logger.printError('WHATAP-614', 'Header parsing error', e, false);
+        }
     }
     /************************************/
     /*   Multi Server Transaction Trace */
@@ -391,21 +424,33 @@ HttpObserver.prototype.__endTransaction = function(error, ctx, req, res) {
         }
     }
 
-    if(param_enabled) {
-        var query = JSON.stringify(req.query || '');
-        var body = JSON.stringify(req.body || '');
-        //var param = JSON.stringify(req.params || '');
-        //console.log(query.length + " // " + body.length);
-        var step = new MessageStep();
-        step.hash = HashUtil.hashFromString("HTTP-PARAMETERS");
-        step.start_time = ctx.getElapsedTime();
-        if(query.length > 2) {
-            step.desc = '[GET] ' + query;
-        } else {
-            step.desc = '[POST] ' + body;
+    if (param_enabled && req.query && Object.keys(req.query).length > 0) {
+        const query = req.query;
+        let secureMsgStep = new SecureMsgStep(ctx.getElapsedTime());
+        secureMsgStep.hash = HashUtil.hashFromString("HTTP-PARAMETERS");
+
+        const profileHttpParameterKeysSet = profile_http_parameter_keys ? new Set(profile_http_parameter_keys.split(',')) : null;
+
+        try {
+            const desc = Object.entries(query)
+                .filter(([key]) => !profileHttpParameterKeysSet || profileHttpParameterKeysSet.has(key))
+                .map(([key, value]) => `${key}=${value}`)
+                .join('\n');
+
+            if (desc) {
+                const crc = { value: 0 };
+                secureMsgStep.value = toParamBytes(desc, crc);
+                secureMsgStep.crc = crc.value;
+
+                DataTextAgent.MESSAGE.add(secureMsgStep.hash, "HTTP-PARAMETERS");
+                ctx.profile.push(secureMsgStep);
+            } else {
+                secureMsgStep = null;
+            }
+        } catch (e) {
+            secureMsgStep = null;
+            Logger.printError('WHATAP-613', 'Parameter parsing error', e, false);
         }
-        DataTextAgent.MESSAGE.add(step.hash, "HTTP-PARAMETERS");
-        ctx.profile.push(step);
     }
 
     if(profile_error_step_enabled && ctx.statusMessage){
@@ -823,7 +868,7 @@ function updateNotFoundIgnoreMap(url, url_hash, currentTime) {
     }
 
     if (ignore_value.count >= 50) {
-        httpc_not_found_ignore_url.push(url);
+        httpc_not_found_ignore_url.add(url);
         httpc_not_found_ignore_map.remove(url_hash);
         Logger.print('WHATAP-610', "The ignore option for URL " + url + " is applied.", false);
     }
@@ -839,4 +884,15 @@ function shouldEndCurrentTransaction(not_found_ignore, ctx, res, requestPath) {
     return !is_ignore;
 }
 
+var toParamBytes = function (p, crc) {
+    if (p == null || p.length === 0) {
+        return null;
+    }
+    try {
+        return ParamSecurity.encrypt(Buffer.from(p, 'utf8'), crc);
+    } catch (e) {
+        return null;
+    }
+};
+
 exports.HttpObserver = HttpObserver;

package/lib/observers/maria-observer.js

@@ -24,6 +24,7 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     DateUtil            = require('../util/dateutil'),
     Buffer              = require('buffer').Buffer,
     MessageStep = require("../step/message-step");
+// const ResourceProfile = require("../util/resourceprofile");
 
 var MariaObserver = function (agent) {
     this.agent = agent;
@@ -145,6 +146,11 @@ var queryHook = function (dbc, agent) {
             sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
             ctx.sql_time += sql_step.elapsed;
 
+            // if(conf.getProperty('profile_sql_resource_enabled', false)){
+            //     sql_step.start_cpu = ResourceProfile.getCPUTime();
+            //     sql_step.start_mem = ResourceProfile.getUsedHeapSize();
+            // }
+
             ctx.footprint('Maria Query Done');
 
             MeterSql.add(dbc_hash, sql_step.elapsed, false);

package/lib/observers/mongoose-observer.js

@@ -136,15 +136,21 @@ MongooseObserver.prototype.inject = function (mod, modName) {
 
                 const mongooseCollection = _ret.mongooseCollection;
                 try {
-                    var sql = mongooseCollection.modelName + ' ' + _ret.op ;//+ ': ' + JSON.stringify(Object.keys(args[0]));
-                    if(_args[0]) { sql += ' field=' + JSON.stringify(Object.keys(_args[0])); }
-                    if(_args[1]) { sql += ' value=' + JSON.stringify(Object.keys(_args[1])); }
-
-                    sql_step.hash = HashUtil.hashFromString(sql);
-                    sql_step.start_time = ctx.getElapsedTime();
-                    sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
-                    DataTextAgent.SQL.add(sql_step.hash, sql);
-                    ctx.profile.push(sql_step);
+                    if(mongooseCollection && mongooseCollection.modelName){
+                        var sql = mongooseCollection.modelName + ' ' + _ret.op;//+ ': ' + JSON.stringify(Object.keys(args[0]));
+                        if (_args[0]) {
+                            sql += ' field=' + JSON.stringify(Object.keys(_args[0]));
+                        }
+                        if (_args[1]) {
+                            sql += ' value=' + JSON.stringify(Object.keys(_args[1]));
+                        }
+
+                        sql_step.hash = HashUtil.hashFromString(sql);
+                        sql_step.start_time = ctx.getElapsedTime();
+                        sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
+                        DataTextAgent.SQL.add(sql_step.hash, sql);
+                        ctx.profile.push(sql_step);
+                    }
                 } catch(e) {
                     Logger.printError("WHATAP-611", "Mongodb query error", e, false);
                     sql_step = null;

package/lib/observers/mssql-observer.js

@@ -22,6 +22,7 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     Logger              = require('../logger');
 const DateUtil = require('../util/dateutil');
 const MessageStep = require("../step/message-step");
+// const ResourceProfile = require("../util/resourceprofile");
 
 var MssqlObserver = function (agent) {
     this.agent = agent;
@@ -126,6 +127,11 @@ MssqlObserver.prototype.inject = function (mod, modName) {
                     StatSql.addSqlTime(ctx.service_hash, sql_step.dbc,
                         sql_step.hash, sql_step.elapsed, args[0] != null, 0);
 
+                    // if(conf.getProperty('profile_sql_resource_enabled', false)){
+                    //     sql_step.start_cpu = ResourceProfile.getCPUTime();
+                    //     sql_step.start_mem = ResourceProfile.getUsedHeapSize();
+                    // }
+
                     if(psql && psql.type == 'S') {
                         var result_step = new ResultSetStep();
                         result_step.dbc = sql_step.dbc;

package/lib/observers/mysql-observer.js

@@ -24,6 +24,7 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     DateUtil = require('../util/dateutil'),
     Buffer = require('buffer').Buffer,
     MessageStep = require("../step/message-step");
+// const ResourceProfile = require('../util/resourceprofile');
 
 var MysqlObserver = function (agent) {
     this.agent = agent;
@@ -167,6 +168,11 @@ var queryHook = function (dbc, agent) {
             sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
             ctx.sql_time += sql_step.elapsed;
 
+            // if(conf.getProperty('profile_sql_resource_enabled', false)){
+            //     sql_step.start_cpu = ResourceProfile.getCPUTime();
+            //     sql_step.start_mem = ResourceProfile.getUsedHeapSize();
+            // }
+
             ctx.footprint('MySql Query Done');
 
             MeterSql.add(dbc_hash, sql_step.elapsed, false);
@@ -360,7 +366,7 @@ MysqlObserver.prototype.inject = function (mod, moduleName) {
 
     var dbc_pool_hash = 0;
     aop.after(mod, ['createPool', 'createPoolCluster'], function (obj, args, ret) {
-        if (dbc_pool_hash == 0) {
+        if (dbc_pool_hash == 0 && dbc === 'mysql://') {
             if (args.length > 0) {
                 var info = args[0];
                 //Open [DatabaseName] connection 메세지 보여줄때 필요함.

package/lib/observers/pgsql-observer.js

@@ -24,6 +24,7 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     Buffer              = require('buffer').Buffer,
     DateUtil = require('../util/dateutil'),
     MessageStep = require("../step/message-step");
+// const ResourceProfile = require("../util/resourceprofile");
 
 var PgSqlObserver = function (agent) {
     this.agent = agent;
@@ -134,6 +135,12 @@ var queryHook = function (dbc_local, agent) {
 
             sql_step.elapsed = ctx.getElapsedTime() - sql_step.start_time;
             ctx.sql_time += sql_step.elapsed;
+
+            // if(conf.getProperty('profile_sql_resource_enabled', false)){
+            //     sql_step.start_cpu = ResourceProfile.getCPUTime();
+            //     sql_step.start_mem = ResourceProfile.getUsedHeapSize();
+            // }
+
             ctx.footprint('PgSql Query Done');
 
             MeterSql.add(dbc_hash, sql_step.elapsed, false);

package/lib/observers/socket.io-observer.js

@@ -4,51 +4,179 @@
  * can be found in the LICENSE file.
  */
 
-var MeterSocketio = require('../counter/meter/meter-socket.io');
+var MeterSocketio = require('../counter/meter/meter-socket.io'),
+    TraceContextManager = require('../trace/trace-context-manager'),
+    SocketStep = require('../step/socket-step'),
+    conf = require('../conf/configure'),
+    IPUtil = require('../util/iputil'),
+    Logger = require('../logger');
+const {Detector: URLPatternDetector} = require("whatap/lib/trace/serviceurl-pattern-detector");
+const HashUtil = require("whatap/lib/util/hashutil");
+const DataTextAgent = require("whatap/lib/data/datatext-agent");
+const ResourceProfile = require("whatap/lib/util/resourceprofile");
+const ProfilePack         = require('../pack/profile-pack');
+const TxRecord            = require('../service/tx-record');
+const DateUtil            = require('../util/dateutil');
+const SecurityMaster      = require('../net/security-master');
+const DataProfileAgent    = require('../data/dataprofile-agent');
+const MessageStep = require("whatap/lib/step/message-step");
+const MeterUsers = require("whatap/lib/counter/meter/meter-users");
+const DataPackSender = require("whatap/lib/data/datapack-sender");
+const MeterService        = require('../counter/meter/meter-service').MeterService;
+
+var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', false);
+conf.on('trace_background_socket_enabled', function (newProps) {
+    trace_background_socket_enabled = newProps;
+})
+var trace_sampling_enabled = conf.getProperty('trace_sampling_enabled', false);
+conf.on('trace_sampling_enabled', function (newProps) {
+    trace_sampling_enabled = newProps;
+})
+var trace_sampling_tps = conf.getProperty('trace_sampling_tps', 1000);
+conf.on('trace_sampling_tps', function (newProps) {
+    trace_sampling_tps = newProps;
+})
 
 var SocketIOObserver = function(agent){
     this.agent = agent;
     this.packages = ['socket.io'];
 };
 
-SocketIOObserver.prototype.inject = function( mod ){
+var socket_count = {
+    count: 0,
+    start_time: null
+};
+
+SocketIOObserver.prototype.inject = function (mod, moduleName) {
+    if (mod.__whatap_observe__) {
+        return;
+    }
+    mod.__whatap_observe__ = true;
+    Logger.initPrint("SocketIOObserver");
+
     var self = this;
     var aop = self.agent.aop;
 
-    if(mod.__whatap_observe__) { return; }
-    mod.__whatap_observe__ = true;
+    aop.after(mod.Server.prototype, 'on', function (obj, args, ret, lctx) {
+        aop.after(mod.Socket.prototype, 'emit', function (_obj, _args, _ret, _lctx) {
+            if(!trace_background_socket_enabled) {
+                return;
+            }
+            if(trace_sampling_enabled){
+                var now = Date.now();
+                if(!socket_count.start_time) {
+                    socket_count.start_time = now;
+                }
 
-    aop.after(mod.prototype, ['listen','attach'], function(obj, args, ret) {
-        if (!ret.sockets) return;
+                if ((now - socket_count.start_time) > 5000) {
+                    socket_count.start_time = now;
+                    socket_count.count = 0;
+                }
 
-        aop.before(ret.sockets, ['on', 'addListener'], function(obj, args) {
-            if (args[0] !== 'connection') return;
+                socket_count.count++;
+                if(socket_count.count > trace_sampling_tps) {
+                    MeterService.add(0, 1, 0, 0, 0, 0);
+                    return;
+                }
+            }
 
-            aop.functionHook(args, -1, function(obj, args) {
-                if (!args[0]) return; //no client socket
+            TraceContextManager._asyncLocalStorage.run(initCtx(_obj, _args), () => {
+                try{
+                    var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                    if (!ctx) {
+                        return;
+                    }
 
-                var socket = args[0];
+                    var socket = _obj;
+                    var host;
+                    if (socket.handshake && socket.handshake.address && socket.handshake.address.includes(':')) {
+                        host = socket.handshake.address.substring(socket.handshake.address.lastIndexOf(':') + 1);
+                    }
 
-                MeterSocketio.connected(1);
+                    ctx.socket_connecting = true;
+                    ctx.footprint('Socket Connecting: ' + host);
 
-                socket.on('disconnect', function() {
-                    MeterSocketio.disconnected();
-                });
+                    var step = new SocketStep();
+                    step.start_time = ctx.getElapsedTime();
+                    step.ipaddr = Buffer.from(IPUtil.stringToBytes(host));
+                    // step.port = port;
 
-                aop.before(socket, ['emit', 'send'], function(obj, args) {
-                    if (args[0] === 'newListener') return;
-                    MeterSocketio.send(null, args);
-                });
+                    ctx.socket_connecting = false;
+                    step.elapsed = ctx.getElapsedTime() - step.start_time;
+                    ctx.profile.push(step);
 
-                aop.before(socket, ['on', 'addListener'], function(obj, args) {
-                    if (args[0] === 'disconnect') return;
-                    aop.functionHook(args, -1, function (obj, args) {
-                        MeterSocketio.receive(null, args[0]);
-                    });
-                });
+                    ctx.footprint('Socket Connecting Done');
+
+                    self.__endTransaction(null, ctx);
+                    return;
+                }catch (e) {
+                    Logger.printError('WHATAP-616', 'socket.io transaction error..', e, false);
+                    return;
+                }
             });
         });
     });
+};
+
+SocketIOObserver.prototype.__endTransaction = function(error, ctx) {
+    try {
+        var profile = new ProfilePack();
+        var wtx = new TxRecord();
+        wtx.endTime = DateUtil.currentTime();
+        profile.time  = wtx.endTime;
+        wtx.elapsed = ctx.getElapsedTime();
+
+        DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+        wtx.seq = ctx.txid;
+        wtx.service = ctx.service_hash;
+        wtx.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
+        wtx.malloc = ResourceProfile.getUsedHeapSize()-ctx.start_malloc;
+        if(wtx.malloc < 0) { wtx.malloc = 0; }
+        wtx.status = 2;
+
+        wtx.ipaddr = ctx.remoteIp;
+
+        MeterService.add(wtx.service, wtx.elapsed,
+            wtx.errorLevel, ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+
+        profile.oid = SecurityMaster.OID;
+        profile.service = wtx;
+
+        TraceContextManager.end(ctx._id);
+
+        setTimeout(function () {
+            DataProfileAgent.sendProfile(ctx, profile, false);
+            TraceContextManager.end(ctx._id);
+            ctx = null;
+        }, 100);
+    } catch (e) {
+        Logger.printError('WHATAP-615', 'Socket.io end transaction error..', e, false);
+        TraceContextManager.end(ctx._id);
+        ctx = null;
+    }
+
+};
+
+function initCtx(socket, args) {
+    const ctx = TraceContextManager.start();
+    if (!ctx) {return;}
+
+    var remote_addr;
+    const address = socket.handshake.address;
+    if(address && address.includes(':')){
+        remote_addr = address.substring(address.lastIndexOf(':')+1);
+    }
+
+    ctx.start_malloc = ResourceProfile.getUsedHeapSize();
+    ctx.start_cpu = ResourceProfile.getCPUTime();
+
+    remote_addr=IPUtil.checkIp4(remote_addr);
+    ctx.remoteIp = IPUtil.stringToInt(remote_addr);
+    ctx.userid = Long.fromNumber(ctx.remoteIp);
+    MeterUsers.add(ctx.userid);
+
+    return ctx;
 }
 
 exports.SocketIOObserver = SocketIOObserver;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "whatap",
   "homepage": "http://www.whatap.io",
-  "version": "0.4.93",
-  "releaseDate": "20240214",
+  "version": "0.4.94",
+  "releaseDate": "20240307",
   "description": "Monitoring and Profiling Service",
   "main": "index.js",
   "scripts": {},