whatap 0.4.98 → 0.5.1

package/lib/conf/config-default.js

@@ -11,7 +11,7 @@ function num(k, defValue){
 };
 function bool(k, defValue){
     var v=process.env[k];
-    return v?Boolean(v):defValue;
+    return v ? (v === 'true') : defValue;
 };
 function str(k, defValue){
     var v=process.env[k];
@@ -154,8 +154,8 @@ var ConfigDefault = {
     "httpc_not_found_ignore": bool('httpc_not_found_ignore', false),
     "httpc_not_found_ignore_time": num('httpc_not_found_ignore', 300000),
     "ignore_sql_error_code_set": str('ignore_sql_error_code_set', ''),
-    "ignore_nextjs_build_file_enabled": bool('ignore_nextjs_build_file_enabled', true),
-    "ignore_nextjs_build_file_path": str('ignore_nextjs_build_file_path', '/_next/'),
+    "ignore_build_file_enabled": bool('ignore_build_file_enabled', true),
+    "ignore_build_file_path": str('ignore_build_file_path', '/_next/'),
 
     //2017.05.02 AUTO ONAME
     "auto_oname_enabled" : bool('auto_oname_enabled',false),
@@ -259,6 +259,13 @@ var ConfigDefault = {
 
     "stat_ipurl_enabled": bool("stat_ipurl_enabled", false),
     "stat_ipurl_max_count": num("stat_ipurl_max_count", 10000),
+
+    "grpc_profile_enabled": bool("grpc_profile_enabled", true),
+    "grpc_profile_stream_client_enabled": bool("grpc_profile_stream_client_enabled", true),
+    "grpc_profile_stream_server_enabled": bool("grpc_profile_stream_server_enabled", true),
+    "grpc_profile_ignore_method": bool("grpc_profile_ignore_method", true),
+
+    "oname_port_postfix_enabled": bool("oname_port_postfix_enabled", false),
 };
 
 ConfigDefault._hook_method_ignore_prefix = ConfigDefault.hook_method_ignore_prefixes.split(',');

package/lib/conf/configure.js

@@ -17,6 +17,7 @@ var DataInputX = require('./../io/data-inputx'),
     StringUtil = require('./../util/string-util'),
     Cypher = require('./../util/cypher'),
     ConfigDefault = require("./config-default"),
+    LogConfigDefault = require("./log-config-default"),
     ConfSysMon = require('./conf-sys-mon'),
     MapValue = require('./../value/map-value'),
     IntKeyLinkedMap = require('./../util/intkey-linkedmap'),
@@ -27,6 +28,7 @@ var Configuration = function() {
     var self = this;
     EventEmitter.call(this);
     this.apply(ConfigDefault);
+    this.apply(LogConfigDefault);
 
     this.inx = 0;
     this._customProps = {};
@@ -65,6 +67,13 @@ Configuration.prototype.reload = function(cb) {
                         p[key] = ConfigDefault[key];
                     }
                 });
+                var logDefKeys = Object.keys(LogConfigDefault);
+                logDefKeys.forEach(function (key) {
+                    if(LogConfigDefault[key]) {
+                        p[key] = LogConfigDefault[key];
+                    }
+                });
+
                 if(process.env.WHATAP_SERVER_HOST){
                     p['whatap.server.host']=process.env.WHATAP_SERVER_HOST;
                 }
@@ -111,8 +120,8 @@ Configuration.prototype.reload = function(cb) {
 
                 oldKeys.forEach(function (key) {
                     if(newKeys.indexOf(key) < 0) {
-                        if(ConfigDefault[key] != null) {
-                            self[key] = ConfigDefault[key];
+                        if(ConfigDefault[key] !== null || LogConfigDefault[key] !== null) {
+                            self[key] = ConfigDefault[key] !== null ? ConfigDefault[key] : LogConfigDefault[key];
                         } else {
                             delete self[key];
                         }
@@ -260,7 +269,7 @@ Configuration.prototype.setProperty = function(key, value) {
     this.emit(key, value);
 };
 Configuration.prototype.getProperty = function(key, defaultValue) {
-    if(!this[key]){
+    if(this[key] === undefined){
         return defaultValue;
     } else{
         return this[key];

package/lib/conf/log-config-default.js

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+function num(k, defValue){
+    var v=process.env[k];
+    return v?Number(v):defValue;
+};
+function bool(k, defValue){
+    var v=process.env[k];
+    return v?Boolean(v):defValue;
+};
+function str(k, defValue){
+    var v=process.env[k];
+    return v?v:defValue;
+};
+var LogConfigDefault = {
+    "logsink_enabled": bool("logsink_enabled", false),
+    "logsink_zip_enabled": bool("logsink_zip_enabled", false),
+    "logsink_category_stdout": "AppStdOut",
+    "logsink_category_stderr": "AppStdErr",
+    "logsink_queue_size": num("logsink_queue_size", 500),
+    "logsink_limit_content_enabled": bool("logsink_limit_content_enabled", true),
+    "logsink_limit_content_length": num("logsink_limit_content_length", 10000),
+    "logsink_max_content_log_enabled": bool("logsink_max_content_log_enabled", true),
+    "logsink_max_content_alert_enabled": bool("logsink_max_content_alert_enabled", false),
+    "logsink_max_content_alert_silent_time": num("logsink_max_content_alert_silent_time", 10000),
+    "logsink_trace_txid_enabled": bool("logsink_trace_txid_enabled", true),
+    "logsink_trace_mtid_enabled": bool("logsink_trace_mtid_enabled", true),
+    "logsink_high_secure_enabled": bool("logsink_high_secure_enabled", false),
+    "debug_logsink_zip_enabled": bool("debug_logsink_zip_enabled", false),
+    "logsink_max_buffer_size": num("logsink_max_buffer_size", 1024*64),
+    "logsink_max_wait_time": num("logsink_max_wait_time", 2000)
+};
+module.exports = LogConfigDefault;

package/lib/control/control-handler.js

@@ -15,7 +15,9 @@ var PackageCtrHelper = require('./packagectr-helper'),
     TextValue = require('../value/text-value'),
     ResourceProfile = require('../util/resourceprofile'),
     TraceContextManager = require('../trace/trace-context-manager'),
-    Logger = require('../logger');
+    Logger = require('../logger'),
+    StatusDetector = require('../topology/status-detector'),
+    BlobValue = require('../value/blob-value');
 
 var ControlHandler = function () {
     this.queue = new RequestQueue(100);
@@ -26,17 +28,17 @@ ControlHandler.prototype.add = function (p /* Pack */) {
     this.process(p);
 };
 
-ControlHandler.prototype.process = function (p) {
+ControlHandler.prototype.process = async function (p) {
     switch (p.getPackType()) {
         case PackEnum.PARAMETER:
-            this.handle(p);
+            await this.handle(p);
             break;
         default:
             Logger.printError('WHATAP-203', 'Unknown pack received ' + p, new Error('Unknown pack'));
     }
 };
 
-ControlHandler.prototype.handle = function (p) {
+ControlHandler.prototype.handle = async function (p) {
     var self = this;
     if (!this._dataPackSender) {
         this._dataPackSender = require('../data/datapack-sender');
@@ -115,7 +117,7 @@ ControlHandler.prototype.handle = function (p) {
             }
             break;
         case ParamDef.LOADED_CLASS_REDEFINE:
-          self._dataPackSender.sendResponseHide(p.getResponse());
+            self._dataPackSender.sendResponseHide(p.getResponse());
             break;
         case ParamDef.NODE_MODULE_DEPENDENCY:
 
@@ -181,7 +183,7 @@ ControlHandler.prototype.handle = function (p) {
                     var key = en.nextElement();
                     var val = mv.getText(key);
                     props[key] = val
-                } 
+                }
                 Configure.saveProperty(props);
             } catch (e) {
                 p.putValue("error", e.toString())
@@ -222,6 +224,17 @@ ControlHandler.prototype.handle = function (p) {
                 self._dataPackSender.sendResponseHide(p.getResponse());
             });
             break;
+        case ParamDef.GET_TOPOLOGY:
+            try {
+                const node = await new StatusDetector().process();
+                if (node) {
+                    p.putValue("node", new BlobValue(node.toBytes()));
+                }
+                this._dataPackSender.sendResponseHide(p.getResponse());
+            } catch (e) {
+                Logger.printError('WHATAP-203', 'Node Module Dependency ', e, true);
+            }
+            break;
     }
 }
 

package/lib/core/agent.js

@@ -29,7 +29,8 @@ var Interceptor      = require('./interceptor').Interceptor,
     ThriftObserver    = require('../observers/thrift-observer').ThriftObserver,
     PromiseObserver  = require('../observers/promise-observer').PromiseObserver,
     PgSqlObserver    = require('../observers/pgsql-observer').PgSqlObserver,
-    ScheduleObserver = require('../observers/schedule-observer').ScheduleObserver;
+    ScheduleObserver = require('../observers/schedule-observer').ScheduleObserver,
+    GRpcObserver     = require('../observers/grpc-observer').GRpcObserver;
 
 var Configuration    = require('./../conf/configure'),
     SecurityMaster   = require('./../net/security-master'),
@@ -102,21 +103,21 @@ NodeAgent.prototype.findRoot = function () {
 };
 NodeAgent.prototype.init = function(cb) {
     var self = this;
-    if(self._initialized) { 
-        return; 
+    if(self._initialized) {
+        return;
     }
     self._initialized = true;
     self.starttime = Date.now();
 
     self.findRoot();
-    
+
     Logger.initializer.process();
     Logger.print('WHATAP-001', 'Start initialize WhaTap Agent... Root[' + self._conf['app.root'] + ']', true);
 
     if(self._conf['app.root'] == null || self._conf['app.root'].length == 0) {
         return Logger.print("WHATAP-001", "Can not find application root directory", true);
     }
-    
+
     NodeUtil.getPackageJson();
     PackageCtrHelper.dynamicHook();
 
@@ -131,7 +132,7 @@ NodeAgent.prototype.init = function(cb) {
             Logger.print("WHATAP-REQLOG" , "ReqLog Init Start !!!! " , false);
             RequestLog.initializer.process();
         }
-        
+
         self._securityMaster.run( function (err) {
             if(err) {
                 Logger.printError('WHATAP-104', 'Failed to connect to whatap server', err, false);
@@ -179,7 +180,7 @@ NodeAgent.prototype.initOK = function(cb) {
         self._counterManager.run();
         PluginLoaderManager.start();
     }
-    
+
     WhatapUtil.printWhatap();
     self.connectCount = 0;
     self.setLoopTime = 5000;
@@ -266,7 +267,8 @@ NodeAgent.prototype.loadObserves = function() {
     observes.push(PromiseObserver);
     observes.push(PgSqlObserver);
     observes.push(ScheduleObserver);
-     
+    // observes.push(GRpcObserver);
+
     var packageToObserve = {};
     observes.forEach(function(observeObj) {
         var observe = new observeObj(agent);

package/lib/core/shimmer.js

@@ -0,0 +1,98 @@
+'use strict'
+
+function isFunction (funktion) {
+    return typeof funktion === 'function'
+}
+
+function defineProperty (obj, name, value) {
+    var enumerable = !!obj[name] && obj.propertyIsEnumerable(name)
+    Object.defineProperty(obj, name, {
+        configurable: true,
+        enumerable: enumerable,
+        writable: true,
+        value: value
+    })
+}
+
+var shimmer = function () {
+}
+
+var wrap = function (nodule, name, wrapper) {
+    if (!nodule || !nodule[name]) {
+        return;
+    }
+
+    if (!wrapper) {
+        return;
+    }
+
+    if (!isFunction(nodule[name]) || !isFunction(wrapper)) {
+        return
+    }
+
+    var original = nodule[name]
+    var wrapped = wrapper(original, name)
+
+    defineProperty(wrapped, '__original', original)
+    defineProperty(wrapped, '__unwrap', function () {
+        if (nodule[name] === wrapped) defineProperty(nodule, name, original)
+    })
+    defineProperty(wrapped, '__wrapped', true)
+
+    defineProperty(nodule, name, wrapped)
+    return wrapped
+}
+
+function unwrap (nodule, name) {
+    if (!nodule || !nodule[name]) {
+        return
+    }
+
+    if (!nodule[name].__unwrap) {
+    } else {
+        return nodule[name].__unwrap()
+    }
+}
+
+function massWrap (nodules, names, wrapper) {
+    if (!nodules) {
+        return
+    } else if (!Array.isArray(nodules)) {
+        nodules = [nodules]
+    }
+
+    if (!(names && Array.isArray(names))) {
+        return
+    }
+
+    nodules.forEach(function (nodule) {
+        names.forEach(function (name) {
+            wrap(nodule, name, wrapper)
+        })
+    })
+}
+
+function massUnwrap (nodules, names) {
+    if (!nodules) {
+        return
+    } else if (!Array.isArray(nodules)) {
+        nodules = [nodules]
+    }
+
+    if (!(names && Array.isArray(names))) {
+        return
+    }
+
+    nodules.forEach(function (nodule) {
+        names.forEach(function (name) {
+            unwrap(nodule, name)
+        })
+    })
+}
+
+shimmer.wrap = wrap;
+shimmer.unwrap = unwrap;
+shimmer.massWrap = massWrap;
+shimmer.massUnwrap = massUnwrap;
+
+module.exports = shimmer;

package/lib/counter/task/agentinfo.js

@@ -96,7 +96,7 @@ AgentInfo.prototype.checkAutoScaleIn = function() {
         var e = new EventPack();
         e.title = "SILENT_SHUTDOWN";
         e.level = EventLevel.INFO;
-        DataPackSender.send(e);
+        DataPackSender.sendPack(e);
     }
 }
 

package/lib/data/datapack-sender.js

@@ -167,6 +167,22 @@ var sendStatUserAgentPack = function(p){
     return sendPack(p);
 };
 
+var sendLogSinkPack = function(p){
+    if (p == null)
+        return;
+
+    p.pcode = secuMaster.PCODE;
+    p.oid = secuMaster.OID;
+    p.okind=secuMaster.OKIND;
+    p.onode=secuMaster.ONODE;
+
+    if (conf.getProperty('logsink_high_secure_enabled', false)) {
+        TcpRequestMgr.add(NetFlag.SECURE_CYPHER, p);
+    }else{
+        TcpRequestMgr.add(NetFlag.SECURE_HIDE, p);
+    }
+};
+
 var sendEventPack = function(p){
     return sendPack(p);
 };
@@ -251,6 +267,7 @@ var DataPackSender = {
     sendStatRemoteIpPack    : sendStatRemoteIpPack,
     sendStatRemoteIpUrlPack : sendStatRemoteIpUrlPack,
     sendStatUserAgentPack   : sendStatUserAgentPack,
+    sendLogSinkPack         : sendLogSinkPack,
 
     sendRealtimeUserPack    : sendRealtimeUserPack,
     sendEventPack           : sendEventPack,

package/lib/data/zipprofile.js

@@ -185,12 +185,12 @@ class ZipProfile {
     }
 
     startProcessQueue() {
-        if (!this.processing) {
+        if (!this.isProcessing) {
             this.processQueue()
         }
     }
     stopProcessQueue() {
-        this.processing = false;
+        this.isProcessing = false;
     }
 }
 

package/lib/logger.js

@@ -168,6 +168,12 @@
              console.log('[WHATAP]Logger Error - WHATAP ROOT DIR IS NULL!!!!!!!!!');
              return;
          }
+         if(conf.getProperty('log_root', null)){
+             root = conf.getProperty('log_root', null);
+             if(fs.existsSync(root) == false) {
+                 fs.mkdirSync(root, {recursive: true});
+             }
+         }
          var dir = path.join(root, 'logs');
          if(fs.existsSync(dir) == false) {
              fs.mkdirSync(dir);
@@ -188,8 +194,14 @@
          if(conf.log_rotation_enabled === false || conf.log_enabled === false) { return; }
          if(conf.log_keep_days <= 0) { return; }
  
+         var root = conf['app.root'];
+         if(conf.getProperty('log_root', null)){
+             root = conf.getProperty('log_root', null);
+             if(fs.existsSync(root) == false) {
+                 fs.mkdirSync(root, {recursive: true});
+             }
+         }
          var nowUnit = DateUtil.getDateUnit(),
-             root = conf['app.root'],
              dir = path.join(root, 'logs'),
              log_prefix = WHATAP_CONF+"-";
  
@@ -233,6 +245,12 @@
  
  
              var root = conf['app.root'];
+             if(conf.getProperty('log_root', null)){
+                 root = conf.getProperty('log_root', null);
+                 if(fs.existsSync(root) == false) {
+                     fs.mkdirSync(root, {recursive: true});
+                 }
+             }
              if(root==null ){
                  return null;
              }
@@ -276,6 +294,12 @@
          var o = new MapValue();
          if(conf.log_file_enabled === false ) { return o; }
          var root = conf['app.root'];
+         if(conf.getProperty('log_root', null)){
+             root = conf.getProperty('log_root', null);
+             if(fs.existsSync(root) == false) {
+                 fs.mkdirSync(root, {recursive: true});
+             }
+         }
          if(root==null){
              return o;
          }

package/lib/logsink/line-log-util.js

@@ -0,0 +1,87 @@
+const conf = require('../conf/configure');
+const EventPack = require('../pack/event-pack');
+const DataPackSender = require('../data/datapack-sender');
+const Logger = require('../logger');
+const EventLevel = require('../data/event-level');
+
+class LineLogUtil {
+    constructor() {
+        this.last_log = Date.now();
+        this.last_alert = Date.now();
+    }
+
+    static getInstance() {
+        if (!LineLogUtil.instance) {
+            LineLogUtil.instance = new LineLogUtil();
+        }
+        return LineLogUtil.instance;
+    }
+
+    checkLogContent(lineLog, orgContent) {
+        if (!orgContent) {
+            return;
+        }
+
+        const len = orgContent.length;
+        let maxLength = conf.getProperty('logsink_limit_content_length', 10000);
+        if(maxLength > 10000){
+            maxLength = 10000;
+        }
+
+        if ((conf.getProperty('logsink_limit_content_enabled', true) && len > maxLength) || len > 10000) {
+            lineLog.content = orgContent.substring(0, maxLength) + " ...(truncated)";
+            lineLog.truncated = true;
+
+            this.addTruncateTag(lineLog);
+
+            const alertMessage = lineLog.category + " truncated (len=" + len + ")";
+            this.logAndAlert(alertMessage);
+        } else {
+            lineLog.content = orgContent;
+        }
+    }
+
+    log(message) {
+        if (!conf.getProperty('logsink_max_content_log_enabled', true)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_log < conf.getProperty('logsink_max_content_log_silent_time', 60000)) {
+            return;
+        }
+        this.last_log = now;
+        Logger.print('WHATAP-101', message, false);
+    }
+
+    alertOverflow(message) {
+        if (!conf.getProperty('logsink_max_content_alert_enabled', false)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_alert < conf.getProperty('logsink_max_content_alert_silent_time', 60000)) {
+            return;
+        }
+        this.last_alert = now;
+
+        const eventPack = new EventPack();
+        eventPack.level = EventLevel.FATAL;
+        eventPack.title = "LOG_TRUNCATED";
+        eventPack.message = "Too big Log: " + message;
+        DataPackSender.sendEventPack(eventPack);
+    }
+
+    logAndAlert(alertMessage) {
+        this.log(alertMessage);
+        this.alertOverflow(alertMessage);
+    }
+
+    addTruncateTag(lineLog) {
+        lineLog.tags.putString("truncated", "true");
+    }
+}
+
+LineLogUtil.instance = null;
+
+module.exports = LineLogUtil;

package/lib/logsink/line-log.js

@@ -0,0 +1,12 @@
+const MapValue = require('../value/map-value');
+
+var LineLog = function () {
+    this.truncated;
+    this.time;
+    this.category;
+    this.tags = new MapValue();
+    this.fields = new MapValue();
+    this.content;
+}
+
+module.exports = LineLog;

package/lib/logsink/log-sender.js

@@ -0,0 +1,78 @@
+var Configure = require('../conf/configure');
+var RequestQueue = require('../util/request-queue');
+var Logger = require('../logger');
+var SenderUtil = require('./sender-util');
+var conf = require("../conf/configure");
+
+var logsink_enabled = Configure.getProperty('logsink_enabled', false);
+var logsink_queue_size = Configure.getProperty('logsink_queue_size', 1000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled) {
+        LogSender.getInstance().startProcessQueue();
+    } else {
+        LogSender.getInstance().stopProcessQueue();
+        LogSender.resetInstance();
+    }
+});
+
+class LogSender {
+    constructor() {
+        if (LogSender.instance) {
+            return LogSender.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.isProcessing = false;
+        LogSender.instance = this;
+
+        if (logsink_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!LogSender.instance) {
+            LogSender.instance = new LogSender();
+        }
+        return LogSender.instance;
+    }
+
+    static resetInstance() {
+        if(LogSender.instance){
+            LogSender.instance = null;
+        }
+    }
+
+    add(log) {
+        this.queue.put(log);
+    }
+
+    async processQueue() {
+        this.isProcessing = true;
+
+        while (this.isProcessing) {
+            var log = await this.queue.get();
+            if(log){
+                try {
+                    var sendUtil = new SenderUtil();
+                    sendUtil.send(log);
+                } catch (error) {
+                    Logger.printError('WHATAP-702', 'Error occurred during send LogSinkPack.' + error, false);
+                }
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue();
+        }
+    }
+
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = LogSender;

package/lib/logsink/log-tracer.js

@@ -0,0 +1,40 @@
+var conf = require('../conf/configure');
+var LineLog = require('./line-log');
+var LineLogUtil = require('./line-log-util');
+var LogSender = require('./log-sender');
+var TraceContextManager = require('../trace/trace-context-manager');
+
+var LogTracer = function () {
+    this.logSender = LogSender.getInstance();
+}
+
+LogTracer.prototype.addStdWrite = function (content, category) {
+    var lineLog = new LineLog();
+    lineLog.category = category
+
+    var lineLogUtil = LineLogUtil.getInstance();
+    lineLogUtil.checkLogContent(lineLog, content);
+
+    if(lineLog.content){
+        this.addTxTag(lineLog);
+        this.logSender.add(lineLog)
+    }
+}
+
+LogTracer.prototype.addTxTag = function (lineLog) {
+    if(!lineLog){
+        return;
+    }
+
+    var ctx = TraceContextManager.getCurrentContext();
+    if(ctx){
+        if(conf.getProperty('logsink_trace_txid_enabled', true) && !ctx.txid.isZero()){
+            lineLog.fields.putLong('@txid', ctx.txid);
+        }
+        if(conf.getProperty('logsink_trace_mtid_enabled', true) && !ctx.mtid.isZero()){
+            lineLog.fields.putLong('@mtid', ctx.mtid);
+        }
+    }
+}
+
+module.exports = LogTracer;