whatap 0.4.94 → 0.4.96

package/lib/conf/config-default.js

@@ -64,6 +64,14 @@ var ConfigDefault = {
     "profile_http_parameter_url_prefix": "/",
     "profile_http_header_url_prefix": "/",
     "profile_connection_open_enabled": true,
+    "profile_zip_enabled": bool('profile_zip_enabled', false),
+    "profile_zip_queue_size": num('profile_zip_queue_size', 500),
+    "profile_zip_max_buffer_size": num('profile_zip_max_buffer_size', 1024*1024),
+    "profile_zip_min_size": num('profile_zip_min_size', 100),
+    "debug_profile_zip_enabled": bool('debug_profile_zip_enabled', false),
+    "debug_profile_zip_interval": num('debug_profile_zip_interval', 5000),
+    "profile_zip_max_wait_time": num('profile_zip_max_wait_time', 1000),
+    "profile_zip_max_wait2_time": num('profile_zip_max_wait2_time', 5000),
 
     "hook_method_patterns": str('hook_method_patterns',""),
     "hook_method_access_public_enabled": true,
@@ -99,7 +107,7 @@ var ConfigDefault = {
     "trace_sql_error_depth": num('trace_sql_error_depth', 50),
     "transaction_status_error_enable": bool("transaction_status_error_enable", true),
 
-    "trace_sampling_enabled": bool("trace_sampling_enabled", false),
+    "trace_sampling_enabled": bool("trace_sampling_enabled", true),
     "trace_sampling_tps": bool("trace_sampling_tps", 1000),
 
     "profile_error_sql_time_max": num("profile_error_sql_time_max", 30000),
@@ -146,6 +154,8 @@ var ConfigDefault = {
     "httpc_not_found_ignore": bool('httpc_not_found_ignore', false),
     "httpc_not_found_ignore_time": num('httpc_not_found_ignore', 300000),
     "ignore_sql_error_code_set": str('ignore_sql_error_code_set', ''),
+    "ignore_nextjs_build_file_enabled": bool('ignore_nextjs_build_file_enabled', true),
+    "ignore_nextjs_build_file_path": str('ignore_nextjs_build_file_path', '/_next/'),
 
     //2017.05.02 AUTO ONAME
     "auto_oname_enabled" : bool('auto_oname_enabled',false),

package/lib/core/agent.js

@@ -19,6 +19,7 @@ var Interceptor      = require('./interceptor').Interceptor,
     MysqlObserver    = require('../observers/mysql-observer').MysqlObserver,
     MariaObserver    = require('../observers/maria-observer').MariaObserver,
     SocketioObserver = require('../observers/socket.io-observer').SocketIOObserver,
+    WebsocketObserver = require('../observers/websocket-observer').WebsocketObserver,
     ProcessObserver  = require('../observers/process-observer').ProcessObserver,
     FileObserver     = require('../observers/file-observer').FileObserver,
     MongoObserver    = require('../observers/mongo-observer').MongoObserver,
@@ -253,6 +254,7 @@ NodeAgent.prototype.loadObserves = function() {
     observes.push(MysqlObserver);
     observes.push(MariaObserver);
     observes.push(SocketioObserver);
+    observes.push(WebsocketObserver);
     observes.push(ExpressObserver);
     observes.push(FileObserver);
     observes.push(MongoObserver);

package/lib/data/dataprofile-agent.js

@@ -4,134 +4,143 @@
  * can be found in the LICENSE file.
  */
 
- var DataPackSender  = require('./datapack-sender'),
- StatTranX       = require('../stat/stat-tranx'),
- StatRemoteIp    = require('../stat/stat-remoteip'),
- StatUserAgent   = require('../stat/stat-useragent'),
- StatTranxDomain   = require('../stat/stat-tx-domain'),
- StatTranxMtCaller   = require('../stat/stat-tx-caller'),
- StatTranxReferer   = require('../stat/stat-tx-referer'),
- conf            = require('../conf/configure'),
- HashUtil            = require('../util/hashutil'),
- EventLevel          = require('../data/event-level'),
- DataTextAgent       = require('../data/datatext-agent'),
- Long = require('long'),
- Logger          = require('../logger');
+var DataPackSender  = require('./datapack-sender'),
+    StatTranX       = require('../stat/stat-tranx'),
+    StatRemoteIp    = require('../stat/stat-remoteip'),
+    StatUserAgent   = require('../stat/stat-useragent'),
+    StatTranxDomain   = require('../stat/stat-tx-domain'),
+    StatTranxMtCaller   = require('../stat/stat-tx-caller'),
+    StatTranxReferer   = require('../stat/stat-tx-referer'),
+    conf            = require('../conf/configure'),
+    HashUtil            = require('../util/hashutil'),
+    EventLevel          = require('../data/event-level'),
+    DataTextAgent       = require('../data/datatext-agent'),
+    Long = require('long'),
+    Logger          = require('../logger');
 
+var ZipProfile = require('./zipprofile');
+var profile_zip_enabled = conf.getProperty('profile_zip_enabled', false);
+conf.on('profile_zip_enabled', function(newProperty) {
+    profile_zip_enabled = newProperty;
+});
 
 var DataProfileAgent = {
- last_reject : 0,
- sendProfile : function (ctx, profile, rejected) {
-     try {
-         StatRemoteIp.incRemoteIp(ctx.remoteIp);
-         StatUserAgent.incUserAgent(ctx.userAgent);
-
-         var transaction = profile.service;
-         if(conf.stat_domain_enabled && ctx.http_host_hash!=0) {
-             StatTranxDomain.add(ctx.http_host_hash, ctx.service_hash, transaction.elapsed, ctx.error != 0);
-         }
-         
-         if(conf.trace_referer_enabled && ctx.referer!=0) {
-             StatTranxReferer.add(ctx.referer, ctx.service_hash, transaction.elapsed, ctx.error != 0);
-         }
-
-         if(conf.stat_mtrace_enabled && ctx.mcaller_pcode!=0) {
-             var key = StatTranxMtCaller.getService(ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_spec, ctx.mcaller_url,ctx.service_hash);
-             transaction.mthis_spec = conf.mtrace_spec_hash
-
-             if (ctx.mcaller_spec && ctx.mcaller_spec!='') {
-                 transaction.mcaller_spec = key.mcaller_spec = HashUtil.hashFromString(ctx.mcaller_spec);
-                 DataTextAgent.MTRACE_SPEC.add(transaction.mcaller_spec, ctx.mcaller_spec);
-             }
-             if (ctx.mcaller_url  && ctx.mcaller_url!='') {
-                 transaction.mcaller_url = key.mcaller_url = HashUtil.hashFromString(ctx.mcaller_url);
-                 DataTextAgent.MTRACE_CALLER_URL.add(transaction.mcaller_url, ctx.mcaller_url);
-             }
-
-             // additional multi transaction
-             //key.url = ctx.service_hash;
-             var tc = StatTranxMtCaller.getService(key);
-             if (tc != null) {
-                 tc.count++;
-                 if (transaction.errorLevel >= EventLevel.WARNING) {
-                     tc.error++;
-                 }
-                 tc.time += transaction.elapsed;
-             }
-
-             // StatTranxMtCaller.add(ctx.mcaller_pcode, 
-             //      ctx.mcaller_okind,
-             //      transaction.mcaller_spec,
-             //      transaction.mcaller_url, 
-             //      ctx.service_hash, 
-             //      transaction.elapsed, 
-             //      ctx.error != 0);
-         }
-
-         var stat = StatTranX.getService(transaction.service);
-         
-         if(stat !== null) {
-             stat.count++;
-             if(transaction.error.isZero() ==false) {
-                 stat.error++;
-             }
-             if(transaction.elapsed < 0) {
-                 transaction.elapsed = 0;
-             }
-
-             stat.actived += ctx.profileActive;
-             stat.time_sum += transaction.elapsed;
-             if (transaction.elapsed > stat.time_max) {
-                 stat.time_max = transaction.elapsed;
-             }
-             
-             switch (transaction.apdex) {
-                case 2:
-                    stat.apdex_satisfied++;
-                    break;
-                case 1:
-                    stat.apdex_tolerated++;
-                    break;
+    last_reject : 0,
+    sendProfile : function (ctx, profile, rejected) {
+        try {
+            StatRemoteIp.incRemoteIp(ctx.remoteIp);
+            StatUserAgent.incUserAgent(ctx.userAgent);
+
+            var transaction = profile.service;
+            if(conf.stat_domain_enabled && ctx.http_host_hash!=0) {
+                StatTranxDomain.add(ctx.http_host_hash, ctx.service_hash, transaction.elapsed, ctx.error != 0);
+            }
+
+            if(conf.trace_referer_enabled && ctx.referer!=0) {
+                StatTranxReferer.add(ctx.referer, ctx.service_hash, transaction.elapsed, ctx.error != 0);
             }
-            if (stat.time_min==0 || transaction.elapsed < stat.time_min) {
-                stat.time_min = transaction.elapsed;
+
+            if(conf.stat_mtrace_enabled && ctx.mcaller_pcode!=0) {
+                var key = StatTranxMtCaller.getService(ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_spec, ctx.mcaller_url,ctx.service_hash);
+                transaction.mthis_spec = conf.mtrace_spec_hash
+
+                if (ctx.mcaller_spec && ctx.mcaller_spec!='') {
+                    transaction.mcaller_spec = key.mcaller_spec = HashUtil.hashFromString(ctx.mcaller_spec);
+                    DataTextAgent.MTRACE_SPEC.add(transaction.mcaller_spec, ctx.mcaller_spec);
+                }
+                if (ctx.mcaller_url  && ctx.mcaller_url!='') {
+                    transaction.mcaller_url = key.mcaller_url = HashUtil.hashFromString(ctx.mcaller_url);
+                    DataTextAgent.MTRACE_CALLER_URL.add(transaction.mcaller_url, ctx.mcaller_url);
+                }
+
+                // additional multi transaction
+                //key.url = ctx.service_hash;
+                var tc = StatTranxMtCaller.getService(key);
+                if (tc != null) {
+                    tc.count++;
+                    if (transaction.errorLevel >= EventLevel.WARNING) {
+                        tc.error++;
+                    }
+                    tc.time += transaction.elapsed;
+                }
+
+                // StatTranxMtCaller.add(ctx.mcaller_pcode,
+                //      ctx.mcaller_okind,
+                //      transaction.mcaller_spec,
+                //      transaction.mcaller_url,
+                //      ctx.service_hash,
+                //      transaction.elapsed,
+                //      ctx.error != 0);
             }
-            stat.time_std += (Long.fromInt(transaction.elapsed) * Long.fromInt(transaction.elapsed));
-            
-             stat.sql_count += transaction.sqlCount;
-             stat.sql_time +=transaction.sqlTime;
-             stat.sql_fetch += ctx.rs_count;
-             stat.sql_fetch_time +=ctx.rs_time;
-
-             stat.httpc_count += transaction.httpcCount;
-             stat.httpc_time +=transaction.httpcTime;
-
-             stat.malloc_sum +=transaction.malloc;
-             stat.cpu_sum +=transaction.cpuTime;
-         
-             if (rejected) {
-                 var now = Date.now();
-                 if (now < this.last_reject + 1000) {
-                     return;
-                 }
-                 this.last_reject = now;
-             } else if (stat.profiled == true    // 이전(5분구간 내)에 프로파일이 수집된점이 있음
-                 && ctx.profileActive == 0       // 액티브 스택을 추적한적이 없음 
-                 && transaction.elapsed < conf.profile_basetime
-                 && transaction.error.isZero()) {
-                 return;
-             }
-             stat.profiled = true;
-         }
-
-         var steps = ctx.profile.getSteps();
-         profile.setProfile(steps);
-         DataPackSender.sendProfilePack(profile);
-         
-     } catch(e) {
-         Logger.printError('WHATAP-199', 'dataprofile-agent error', e);
-     }
- }
+
+            var stat = StatTranX.getService(transaction.service);
+
+            if(stat !== null) {
+                stat.count++;
+                if(transaction.error.isZero() ==false) {
+                    stat.error++;
+                }
+                if(transaction.elapsed < 0) {
+                    transaction.elapsed = 0;
+                }
+
+                stat.actived += ctx.profileActive;
+                stat.time_sum += transaction.elapsed;
+                if (transaction.elapsed > stat.time_max) {
+                    stat.time_max = transaction.elapsed;
+                }
+
+                switch (transaction.apdex) {
+                    case 2:
+                        stat.apdex_satisfied++;
+                        break;
+                    case 1:
+                        stat.apdex_tolerated++;
+                        break;
+                }
+                if (stat.time_min==0 || transaction.elapsed < stat.time_min) {
+                    stat.time_min = transaction.elapsed;
+                }
+                stat.time_std += (Long.fromInt(transaction.elapsed) * Long.fromInt(transaction.elapsed));
+
+                stat.sql_count += transaction.sqlCount;
+                stat.sql_time +=transaction.sqlTime;
+                stat.sql_fetch += ctx.rs_count;
+                stat.sql_fetch_time +=ctx.rs_time;
+
+                stat.httpc_count += transaction.httpcCount;
+                stat.httpc_time +=transaction.httpcTime;
+
+                stat.malloc_sum +=transaction.malloc;
+                stat.cpu_sum +=transaction.cpuTime;
+
+                if (rejected) {
+                    var now = Date.now();
+                    if (now < this.last_reject + 1000) {
+                        return;
+                    }
+                    this.last_reject = now;
+                } else if (stat.profiled == true    // 이전(5분구간 내)에 프로파일이 수집된점이 있음
+                    && ctx.profileActive == 0       // 액티브 스택을 추적한적이 없음
+                    && transaction.elapsed < conf.profile_basetime
+                    && transaction.error.isZero()) {
+                    return;
+                }
+                stat.profiled = true;
+            }
+
+            var steps = ctx.profile.getSteps();
+            profile.setProfile(steps);
+            if(profile_zip_enabled){
+                ZipProfile.getInstance().add(profile);
+            } else {
+                DataPackSender.sendProfilePack(profile);
+            }
+
+        } catch(e) {
+            Logger.printError('WHATAP-199', 'dataprofile-agent error', e);
+        }
+    }
 };
 
 module.exports = DataProfileAgent;

package/lib/data/zipprofile.js

@@ -0,0 +1,197 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../conf/configure'),
+    secuMaster      = require('../net/security-master'),
+    TcpRequestMgr   = require('../net/tcprequest-mgr'),
+    DateUtil        = require('./../util/dateutil'),
+    RequestQueue    = require('../util/request-queue'),
+    DataPackSender = require('../data/datapack-sender'),
+    DataOutputX = require('../io/data-outputx'),
+    ZipPack = require('../pack/zip-pack'),
+    DataPackSender = require('../data/datapack-sender'),
+    Logger          = require('../logger');
+
+const zlib = require('zlib');
+
+var profile_zip_enabled = conf.getProperty('profile_zip_enabled', false);
+var profile_zip_max_buffer_size = conf.getProperty('profile_zip_max_buffer_size', 1024*1024);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var profile_zip_queue_size = conf.getProperty('profile_zip_queue_size', 500);
+var profile_zip_max_wait_time = conf.getProperty('profile_zip_max_wait_time', 1000);
+var net_send_max_bytes = conf.getProperty('net_send_max_bytes', 5 * 1024 * 1024);
+var debug_profile_zip_enabled = conf.getProperty('debug_profile_zip_enabled', false);
+var debug_profile_zip_interval = conf.getProperty('debug_profile_zip_interval', 5000);
+conf.on('profile_zip_enabled', function(newProperty) {
+    profile_zip_enabled = newProperty;
+    if (profile_zip_enabled) {
+        ZipProfile.getInstance().startProcessQueue();
+    } else {
+        ZipProfile.getInstance().stopProcessQueue();
+        ZipProfile.resetInstance();
+    }
+});
+
+class ZipProfile {
+    constructor() {
+        if (ZipProfile.instance) {
+            return ZipProfile.instance;
+        }
+        this.queue = new RequestQueue(profile_zip_queue_size);
+        this.no_zip_sent = 0;
+        this.zip_sent = 0;
+        this.last_log = null;
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipProfile.instance = this;
+
+        if (profile_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipProfile.instance) {
+            ZipProfile.instance = new ZipProfile();
+        }
+        return ZipProfile.instance;
+    }
+
+    static resetInstance() {
+        if(ZipProfile.instance){
+            ZipProfile.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendPack(p);
+            this.no_zip_sent++;
+        }
+    }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!profile_zip_enabled) { break; }
+
+            const p = await this.queue.getByTimeout(profile_zip_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        if(b.length >= net_send_max_bytes){ return; }
+
+        this.buffer = Buffer.concat([this.buffer, b]);
+        this.packCount++;
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if(this.buffer.length >= profile_zip_max_buffer_size){
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= profile_zip_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('profile_zip_max_wait_time', 5000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(debug_profile_zip_enabled){
+            if(debug_profile_zip_interval){
+                Logger.print('WHATAP-ZIP-DEBUG', `PROFILE status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length} queue=${this.queue.size()}`, false)
+            } else {
+                this.zip_sent++;
+                var now = Date.now();
+                if(now > (this.last_log + debug_profile_zip_interval)){
+                    this.last_log = now;
+                    this.log(p);
+                    this.zip_sent = 0;
+                }
+            }
+        }
+
+        p.pcode = secuMaster.PCODE;
+        p.oid = secuMaster.OID;
+        p.okind = conf.OKIND;
+        p.onode = conf.ONODE;
+
+        TcpRequestMgr.addProfile(0, p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    log(p){
+        try{
+            var sb = `PROFILE `;
+            sb += `zip_sent=${this.zip_sent}`;
+            sb += ` records=${p.records}`;
+            sb += ` | =${this.buffer.length} => ${p.records.length}`;
+            sb += ` queue=${this.queue.size()}`;
+            if (this.no_zip_sent > 0) {
+                sb += ` no_zip_sent=${this.no_zip_sent}`;
+            }
+            Logger.print('WHATAP-ZIP-DEBUG', sb, false);
+        }catch (e) {
+        }
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (e) {
+            Logger.print('WHATAP-ZIP-ERROR', 'Error occurred during compression.', false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.processing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.processing = false;
+    }
+}
+
+module.exports = ZipProfile;

package/lib/observers/http-observer.js

@@ -59,6 +59,8 @@ var httpc_not_found_ignore_time = conf.getProperty('httpc_not_found_ignore_time'
 var profile_http_header_ignore_keys = conf.getProperty('profile_http_header_ignore_keys', 'Cookie,cookie,accept,user-agent,referer');
 var profile_http_parameter_enabled = conf.getProperty('profile_http_parameter_enabled', true);
 var profile_http_parameter_keys = conf.getProperty('profile_http_parameter_keys', '');
+var ignore_nextjs_build_file_enabled = conf.getProperty('ignore_nextjs_build_file_enabled', true);
+var ignore_nextjs_build_file_path = conf.getProperty('ignore_nextjs_build_file_path', '/_next/');
 conf.on('trace_http_client_ip_header_key', function(newProperty) {
     configIpHeaderKey = newProperty;
 });
@@ -116,6 +118,12 @@ conf.on('profile_http_parameter_enabled', function (newProps) {
 conf.on('profile_http_parameter_keys', function (newProps) {
     profile_http_parameter_keys = newProps;
 })
+conf.on('ignore_nextjs_build_file_enabled', function (newProps) {
+    ignore_nextjs_build_file_enabled = newProps;
+})
+conf.on('ignore_nextjs_build_file_path', function (newProps) {
+    ignore_nextjs_build_file_path = newProps;
+})
 var staticConents = function (newProps) {
     var x=new Set();
     var words = !newProps?[]:newProps.split(',');
@@ -231,8 +239,11 @@ function isStatic(u){
 }
 function initCtx(req, res) {
     /*url이 없으면 추적하지 않는다*/
-    if(req.url == null) { return null; }
+    if(!req.url) { return null; }
     if(ignore_http_method && ignore_http_method.toUpperCase().split(',').includes(req.method)) { return null; }
+    if (ignore_nextjs_build_file_enabled && ignore_nextjs_build_file_path && ignore_nextjs_build_file_path.split(',').some(path => req.url.startsWith(path))) {
+        return null;
+    }
 
     var ctx = TraceContextManager.start();
     if(ctx == null) { return null; }
@@ -738,6 +749,7 @@ HttpObserver.prototype.inject = function( mod, moduleName ) {
 
             var ctx = lctx.context;
             if (ctx == null || ( args[0].host == null && args[0].hostname == null)) { return; }
+            var is_ignore_error = false;
 
             ret.on('response', function(response){
                 var statusCode = response.statusCode;
@@ -757,7 +769,7 @@ HttpObserver.prototype.inject = function( mod, moduleName ) {
                 if (TraceContextManager.resume(ctx._id) == null) { return; }
                 ctx.is_httpc_error = true;
 
-                if (transaction_status_error_enable && step.error.isZero()) {
+                if (transaction_status_error_enable && step.error.isZero() && !is_ignore_error) {
                     step.error = StatError.addError(err.code, err.message, ctx.service_hash,
                         TextTypes.HTTPC_URL, step.url);
                     if (ctx.error.isZero()) {
@@ -807,16 +819,17 @@ HttpObserver.prototype.inject = function( mod, moduleName ) {
 
                     socket.on('timeout', function () {
                         if (TraceContextManager.resume(ctx._id) == null) { return; }
-                        if (step.error.isZero()) {
-                            var msgObj = { 'class': 'Timeout', 'msg': 'Timeout' };
-                            step.error = StatError.addError('Timeout','Timeout', ctx.service_hash);
-                            if (transaction_status_error_enable && ctx.error.isZero()) {
-                                ctx.error = step.error;
-                                ctx.statusCode = 'Timeout';
-                                ctx.statusMessage = 'Timeout';
-                            }
-                        }
-                        endHttpc(ctx, step);
+                        is_ignore_error = true;
+                        // if (step.error.isZero()) {
+                        //     var msgObj = { 'class': 'Timeout', 'msg': 'Timeout' };
+                        //     step.error = StatError.addError('Timeout','Timeout', ctx.service_hash);
+                        //     if (transaction_status_error_enable && ctx.error.isZero()) {
+                        //         ctx.error = step.error;
+                        //         ctx.statusCode = 'Timeout';
+                        //         ctx.statusMessage = 'Timeout';
+                        //     }
+                        // }
+                        // endHttpc(ctx, step);
                     });
                 }
             });

package/lib/observers/socket.io-observer.js

@@ -28,7 +28,7 @@ var trace_background_socket_enabled = conf.getProperty('trace_background_socket_
 conf.on('trace_background_socket_enabled', function (newProps) {
     trace_background_socket_enabled = newProps;
 })
-var trace_sampling_enabled = conf.getProperty('trace_sampling_enabled', false);
+var trace_sampling_enabled = conf.getProperty('trace_sampling_enabled', true);
 conf.on('trace_sampling_enabled', function (newProps) {
     trace_sampling_enabled = newProps;
 })

package/lib/observers/websocket-observer.js

@@ -0,0 +1,195 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var MeterSocketio = require('../counter/meter/meter-socket.io'),
+    TraceContextManager = require('../trace/trace-context-manager'),
+    SocketStep = require('../step/socket-step'),
+    conf = require('../conf/configure'),
+    IPUtil = require('../util/iputil'),
+    Logger = require('../logger');
+const {Detector: URLPatternDetector} = require("whatap/lib/trace/serviceurl-pattern-detector");
+const HashUtil = require("whatap/lib/util/hashutil");
+const DataTextAgent = require("whatap/lib/data/datatext-agent");
+const ResourceProfile = require("whatap/lib/util/resourceprofile");
+const ProfilePack         = require('../pack/profile-pack');
+const TxRecord            = require('../service/tx-record');
+const DateUtil            = require('../util/dateutil');
+const SecurityMaster      = require('../net/security-master');
+const DataProfileAgent    = require('../data/dataprofile-agent');
+const MessageStep = require("whatap/lib/step/message-step");
+const MeterUsers = require("whatap/lib/counter/meter/meter-users");
+const DataPackSender = require("whatap/lib/data/datapack-sender");
+const MeterService        = require('../counter/meter/meter-service').MeterService;
+
+var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', false);
+conf.on('trace_background_socket_enabled', function (newProps) {
+    trace_background_socket_enabled = newProps;
+})
+var trace_sampling_enabled = conf.getProperty('trace_sampling_enabled', true);
+conf.on('trace_sampling_enabled', function (newProps) {
+    trace_sampling_enabled = newProps;
+})
+var trace_sampling_tps = conf.getProperty('trace_sampling_tps', 1000);
+conf.on('trace_sampling_tps', function (newProps) {
+    trace_sampling_tps = newProps;
+})
+
+var WebsocketObserver = function(agent){
+    this.agent = agent;
+    this.packages = ['websocket'];
+};
+
+var socket_count = {
+    count: 0,
+    start_time: null
+};
+
+WebsocketObserver.prototype.inject = function (mod, moduleName) {
+    if (mod.__whatap_observe__) {
+        return;
+    }
+    mod.__whatap_observe__ = true;
+    Logger.initPrint("WebsocketObserver");
+
+    var self = this;
+    var aop = self.agent.aop;
+
+    aop.after(mod.server.prototype, 'on', function (obj, args, ret, lctx){
+        aop.after(mod.connection.prototype, ['send', 'sendUTF', 'sendBytes'], function (obj, args, ret, lctx) {
+            if(!trace_background_socket_enabled) {
+                return;
+            }
+            if(trace_sampling_enabled){
+                var now = Date.now();
+                if(!socket_count.start_time) {
+                    socket_count.start_time = now;
+                }
+
+                if ((now - socket_count.start_time) > 5000) {
+                    socket_count.start_time = now;
+                    socket_count.count = 0;
+                }
+
+                socket_count.count++;
+                if(socket_count.count > trace_sampling_tps) {
+                    MeterService.add(0, 1, 0, 0, 0, 0);
+                    return;
+                }
+            }
+
+            var protocol = obj.protocol;
+            var remoteAddress = obj.remoteAddress;
+
+            TraceContextManager._asyncLocalStorage.run(initCtx(obj, args), () => {
+                try{
+                    var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                    if (!ctx) {
+                        return;
+                    }
+
+                    var host;
+                    if (remoteAddress && remoteAddress.includes(':')) {
+                        host = remoteAddress.substring(remoteAddress.lastIndexOf(':') + 1);
+                    }
+
+                    ctx.socket_connecting = true;
+                    ctx.footprint('Websocket Connecting: ' + host);
+
+                    if (protocol) {
+                        var protocol_hash = HashUtil.hashFromString('Protocol');
+                        var step = new MessageStep();
+                        step.hash = protocol_hash;
+                        step.start_time = ctx.getElapsedTime();
+                        step.desc = protocol;
+
+                        DataTextAgent.MESSAGE.add(protocol_hash, 'Protocol');
+                        ctx.profile.add(step);
+                    }
+
+                    var step = new SocketStep();
+                    step.start_time = ctx.getElapsedTime();
+                    step.ipaddr = Buffer.from(IPUtil.stringToBytes(host));
+                    // step.port = port;
+
+                    ctx.socket_connecting = false;
+                    step.elapsed = ctx.getElapsedTime() - step.start_time;
+                    ctx.profile.push(step);
+
+                    ctx.footprint('Websocket Connecting Done');
+
+                    self.__endTransaction(null, ctx);
+                    return;
+                }catch (e) {
+                    Logger.printError('WHATAP-616', 'Websocket transaction error..', e, false);
+                    return;
+                }
+            });
+        })
+    })
+};
+
+WebsocketObserver.prototype.__endTransaction = function(error, ctx) {
+    try {
+        var profile = new ProfilePack();
+        var wtx = new TxRecord();
+        wtx.endTime = DateUtil.currentTime();
+        profile.time  = wtx.endTime;
+        wtx.elapsed = ctx.getElapsedTime();
+
+        DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+        wtx.seq = ctx.txid;
+        wtx.service = ctx.service_hash;
+        wtx.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
+        wtx.malloc = ResourceProfile.getUsedHeapSize()-ctx.start_malloc;
+        if(wtx.malloc < 0) { wtx.malloc = 0; }
+        wtx.status = 2;
+
+        wtx.ipaddr = ctx.remoteIp;
+
+        MeterService.add(wtx.service, wtx.elapsed,
+            wtx.errorLevel, ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+
+        profile.oid = SecurityMaster.OID;
+        profile.service = wtx;
+
+        TraceContextManager.end(ctx._id);
+
+        setTimeout(function () {
+            DataProfileAgent.sendProfile(ctx, profile, false);
+            TraceContextManager.end(ctx._id);
+            ctx = null;
+        }, 100);
+    } catch (e) {
+        Logger.printError('WHATAP-615', 'Websocket end transaction error..', e, false);
+        TraceContextManager.end(ctx._id);
+        ctx = null;
+    }
+
+};
+
+function initCtx(socket, args) {
+    const ctx = TraceContextManager.start();
+    if (!ctx) {return;}
+
+    var remote_addr;
+    const address = socket.remoteAddress;
+    if(address && address.includes(':')){
+        remote_addr = address.substring(address.lastIndexOf(':')+1);
+    }
+
+    ctx.start_malloc = ResourceProfile.getUsedHeapSize();
+    ctx.start_cpu = ResourceProfile.getCPUTime();
+
+    remote_addr=IPUtil.checkIp4(remote_addr);
+    ctx.remoteIp = IPUtil.stringToInt(remote_addr);
+    ctx.userid = Long.fromNumber(ctx.remoteIp);
+    MeterUsers.add(ctx.userid);
+
+    return ctx;
+}
+
+exports.WebsocketObserver = WebsocketObserver;

package/lib/pack/packenum.js

@@ -4,58 +4,60 @@
  * can be found in the LICENSE file.
  */
 
- exports.PARAMETER   = 0x0100;
- exports.COUNTER_1   = 0x0201;
- exports.PROFILE     = 0x0300;
- exports.ACTIVESTACK_1 = 0x0401;
- exports.TEXT        = 0x0700;
- exports.ERROR_SNAP  = 0x0800;
- exports.ERROR_SNAP_1  = 0x0801;
- exports.REALTIME_USER_1 = 0x0f01;
- 
- exports.STAT_SERVICE= 0x0900;
- exports.STAT_SERVICE_1 = 0x0901;
- exports.STAT_GENERAL= 0x0910;
- exports.STAT_SQL    = 0x0a00;
- exports.STAT_HTTPC  = 0x0b00;
- exports.STAT_ERROR  = 0x0c00;
- exports.STAT_METHOD = 0x0e00;
- exports.STAT_TOP_SERVICE= 0x1000;
- exports.STAT_REMOTE_IP  = 0x1100;
- exports.STAT_USER_AGENT = 0x1200;
- 
- exports.EVENT       = 0x1400;
- exports.HITMAP      = 0x1500;
- exports.HITMAP_1    = 0x1501;
- exports.COUNTER5M   = 0x1600;
- exports.HITMAP5M    = 0x1700;
- 
- exports.TAG_COUNT    = 0x1601;
- 
- exports.PACK_NAME = {
-     0x0100 : 'PARAMETER',
-     0x0201 : 'COUNTER_1',
-     0x0300 : 'PROFILE',
-     0x0401 : 'ACTIVESTACK_1',
-     0x0700 : 'TEXT',
-     0x0800 : 'ERROR_SNAP',
-     0x0801 : 'ERROR_SNAP_1',
-     0x0f01 : 'REALTIME_USER_1',
-     0x0900 : 'STAT_SERVICE',
-     0x0901 : 'STAT_SERVICE_1',
-     0x0910 : 'STAT_GENERAL',
-     0x0a00 : 'STAT_SQL',
-     0x0b00 : 'STAT_HTTPC',
-     0x0c00 : 'STAT_ERROR',
-     0x0e00 : 'STAT_METHOD',
-     0x1000 : 'STAT_TOP_SERVICE',
-     0x1100 : 'STAT_REMOTE_IP',
-     0x1200 : 'STAT_USER_AGENT',
-     0x1400 : 'EVENT',
-     0x1500 : 'HITMAP',
-     0x1501 : 'HITMAP_1',
-     0x1600 : 'COUNTER5M',
-     0x1700 : 'HITMAP5M',
-     0x1601 : 'TAG_COUNT'
- };
+exports.PARAMETER   = 0x0100;
+exports.COUNTER_1   = 0x0201;
+exports.PROFILE     = 0x0300;
+exports.ACTIVESTACK_1 = 0x0401;
+exports.TEXT        = 0x0700;
+exports.ERROR_SNAP  = 0x0800;
+exports.ERROR_SNAP_1  = 0x0801;
+exports.REALTIME_USER_1 = 0x0f01;
+
+exports.STAT_SERVICE= 0x0900;
+exports.STAT_SERVICE_1 = 0x0901;
+exports.STAT_GENERAL= 0x0910;
+exports.STAT_SQL    = 0x0a00;
+exports.STAT_HTTPC  = 0x0b00;
+exports.STAT_ERROR  = 0x0c00;
+exports.STAT_METHOD = 0x0e00;
+exports.STAT_TOP_SERVICE= 0x1000;
+exports.STAT_REMOTE_IP  = 0x1100;
+exports.STAT_USER_AGENT = 0x1200;
+
+exports.EVENT       = 0x1400;
+exports.HITMAP      = 0x1500;
+exports.HITMAP_1    = 0x1501;
+exports.COUNTER5M   = 0x1600;
+exports.HITMAP5M    = 0x1700;
+
+exports.TAG_COUNT   = 0x1601;
+exports.ZIP         = 0x170b;
+
+exports.PACK_NAME = {
+    0x0100 : 'PARAMETER',
+    0x0201 : 'COUNTER_1',
+    0x0300 : 'PROFILE',
+    0x0401 : 'ACTIVESTACK_1',
+    0x0700 : 'TEXT',
+    0x0800 : 'ERROR_SNAP',
+    0x0801 : 'ERROR_SNAP_1',
+    0x0f01 : 'REALTIME_USER_1',
+    0x0900 : 'STAT_SERVICE',
+    0x0901 : 'STAT_SERVICE_1',
+    0x0910 : 'STAT_GENERAL',
+    0x0a00 : 'STAT_SQL',
+    0x0b00 : 'STAT_HTTPC',
+    0x0c00 : 'STAT_ERROR',
+    0x0e00 : 'STAT_METHOD',
+    0x1000 : 'STAT_TOP_SERVICE',
+    0x1100 : 'STAT_REMOTE_IP',
+    0x1200 : 'STAT_USER_AGENT',
+    0x1400 : 'EVENT',
+    0x1500 : 'HITMAP',
+    0x1501 : 'HITMAP_1',
+    0x1600 : 'COUNTER5M',
+    0x1700 : 'HITMAP5M',
+    0x1601 : 'TAG_COUNT',
+    0x170b : 'ZIP'
+};
  

package/lib/pack/zip-pack.js

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var PackEnum = require('./packenum'),
+    Pack = require('./pack'),
+    TxRecord = require('../service/tx-record'),
+    Step = require('../step/step'),
+    DataOutputX = require('../io/data-outputx'),
+    DataInputX = require('../io/data-inputx');
+var zlib = require('zlib');
+
+function ZipPack() {
+    Pack.call(this);
+    this.records = [];
+    this.recordCount = null;
+    this.status = 0;
+}
+ZipPack.prototype = new Pack();
+ZipPack.prototype.constructor = ZipPack;
+ZipPack.prototype.toString = function() {
+    var text = 'ZipPack ' + Pack.prototype.toString.call(this);
+    return text;
+};
+ZipPack.prototype.getPackType = function () {
+    return PackEnum.ZIP;
+};
+ZipPack.prototype.write = function (dout) {
+    Pack.prototype.write.call(this, dout);
+    dout.writeByte(this.status);
+    dout.writeDecimal(this.recordCount);
+    dout.writeBlob(this.records);
+}
+ZipPack.prototype.read = function (din) {
+    Pack.prototype.read.call(this, din);
+    this.status = din.readByte();
+    this.recordCount = din.readDecimal();
+    this.records = din.readBlob();
+    return this;
+}
+ZipPack.prototype.setRecords = function (items, size) {
+    if(size) this.recordCount = size;
+    else this.recordCount = items.length;
+
+    var o = new DataOutputX();
+    for(let i = 0; i < items.length; i++){
+        o.writePack(items[i]);
+    }
+    this.records = o.toByteArray();
+    return this;
+}
+ZipPack.prototype.getRecords = function () {
+    var items = [];
+    if(!this.records) {
+        return null;
+    }
+    var dataInputX = new DataInputX(this.records);
+    for(let i = 0; i < this.recordCount; i++){
+        var p = dataInputX.readPack();
+        p.pcode = this.pcode;
+        p.oid = this.oid;
+        p.okind = this.okind;
+        p.onode = this.onode;
+        items.push(p);
+    }
+    return items;
+}
+
+module.exports = ZipPack;

package/lib/trace/trace-context-manager.js

@@ -160,6 +160,8 @@ TraceContextManager.prototype.endTrace = function (ctx) {
     service.endTime = DateUtil.currentTime();
     service.elapsed = ctx.getElapsedTime();
     service.service = ctx.service_hash;
+    DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
     service.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
     service.malloc = ResourceProfile.getUsedHeapSize() - ctx.start_malloc;
     if(service.malloc < 0) { service.malloc = 0; }

package/lib/util/paramsecurity.js

@@ -20,7 +20,7 @@ ParamSecurity.prototype.getKey = function () {
     return buf;
 };
 
-ParamSecurity.prototype.reload = function () {    
+ParamSecurity.prototype.reload = function () {
     var home = conf['app.root'];
     if (home == null ) {
         var self = this;
@@ -30,7 +30,7 @@ ParamSecurity.prototype.reload = function () {
         return;
     }
 
-    var file = path.join(home, 'paramkey.txt');
+    var file = path.join(home, 'security.conf');
     var self = this;
     try {
         var stat = fs.statSync(file);
@@ -40,8 +40,8 @@ ParamSecurity.prototype.reload = function () {
         }
         var readable = fs.createReadStream(file, {flags : 'r'});
         readable.on('readable', function () {
-            if(self.key !== null 
-                && self.key !== undefined 
+            if(self.key !== null
+                && self.key !== undefined
                 && self.key.length === 0 ) { return; }
             var chunk;
             while (null !== (chunk = readable.read())) {
@@ -50,11 +50,12 @@ ParamSecurity.prototype.reload = function () {
             self.key = Buffer.from(self.key);
         });
     } catch (e) {
-        self.key = self.getKey();
+        // self.key = self.getKey();
+        self.key = 'WHATAP';
         var printWriter = fs.createWriteStream(file, {flags : 'w'});
         printWriter.write(self.key);
         self.key = Buffer.from(self.key);
-    }                                                               
+    }
 };
 ParamSecurity.prototype.encrypt = function (b, crc) {
     var self = this;

package/lib/util/request-queue.js

@@ -17,6 +17,33 @@ RequestQueue.prototype.get = function () {
     }
     return null;
 };
+RequestQueue.prototype.getByTimeout = async function (timeout) {
+    try{
+        if (this.queue.size() > 0) {
+            return this.queue.removeFirst()
+        }
+        var timeto = Date.now() + timeout;
+        var time = timeout;
+        while (this.queue.size() === 0) {
+            try {
+                if (time > 0) {
+                    await new Promise(resolve => setTimeout(resolve, time));
+                }
+            } catch (e) {
+            }
+            time = timeto - Date.now();
+            if (time <= 0) {
+                break;
+            }
+        }
+        if (this.queue.size() > 0) {
+            return this.queue.removeFirst();
+        }
+        return null;
+    }catch (e) {
+        return null;
+    }
+};
 RequestQueue.prototype.put = function (o) {
     if(this.capacity <= 0 || this.queue.size() < this.capacity) {
         this.queue.add(o);
@@ -34,10 +61,10 @@ RequestQueue.prototype.clear = function () {
 };
 
 var failed = function (v) {
-    
+
 };
 var overflowed = function (v) {
-    
+
 };
 
 module.exports = RequestQueue;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "whatap",
   "homepage": "http://www.whatap.io",
-  "version": "0.4.94",
-  "releaseDate": "20240307",
+  "version": "0.4.96",
+  "releaseDate": "20240530",
   "description": "Monitoring and Profiling Service",
   "main": "index.js",
   "scripts": {},