whatap 0.4.97 → 0.5.0

package/lib/logsink/line-log-util.js

@@ -0,0 +1,87 @@
+const conf = require('../conf/configure');
+const EventPack = require('../pack/event-pack');
+const DataPackSender = require('../data/datapack-sender');
+const Logger = require('../logger');
+const EventLevel = require('../data/event-level');
+
+class LineLogUtil {
+    constructor() {
+        this.last_log = Date.now();
+        this.last_alert = Date.now();
+    }
+
+    static getInstance() {
+        if (!LineLogUtil.instance) {
+            LineLogUtil.instance = new LineLogUtil();
+        }
+        return LineLogUtil.instance;
+    }
+
+    checkLogContent(lineLog, orgContent) {
+        if (!orgContent) {
+            return;
+        }
+
+        const len = orgContent.length;
+        let maxLength = conf.getProperty('logsink_limit_content_length', 10000);
+        if(maxLength > 10000){
+            maxLength = 10000;
+        }
+
+        if ((conf.getProperty('logsink_limit_content_enabled', true) && len > maxLength) || len > 10000) {
+            lineLog.content = orgContent.substring(0, maxLength) + " ...(truncated)";
+            lineLog.truncated = true;
+
+            this.addTruncateTag(lineLog);
+
+            const alertMessage = lineLog.category + " truncated (len=" + len + ")";
+            this.logAndAlert(alertMessage);
+        } else {
+            lineLog.content = orgContent;
+        }
+    }
+
+    log(message) {
+        if (!conf.getProperty('logsink_max_content_log_enabled', true)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_log < conf.getProperty('logsink_max_content_log_silent_time', 60000)) {
+            return;
+        }
+        this.last_log = now;
+        Logger.print('WHATAP-101', message, false);
+    }
+
+    alertOverflow(message) {
+        if (!conf.getProperty('logsink_max_content_alert_enabled', false)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_alert < conf.getProperty('logsink_max_content_alert_silent_time', 60000)) {
+            return;
+        }
+        this.last_alert = now;
+
+        const eventPack = new EventPack();
+        eventPack.level = EventLevel.FATAL;
+        eventPack.title = "LOG_TRUNCATED";
+        eventPack.message = "Too big Log: " + message;
+        DataPackSender.sendEventPack(eventPack);
+    }
+
+    logAndAlert(alertMessage) {
+        this.log(alertMessage);
+        this.alertOverflow(alertMessage);
+    }
+
+    addTruncateTag(lineLog) {
+        lineLog.tags.putString("truncated", "true");
+    }
+}
+
+LineLogUtil.instance = null;
+
+module.exports = LineLogUtil;

package/lib/logsink/line-log.js

@@ -0,0 +1,12 @@
+const MapValue = require('../value/map-value');
+
+var LineLog = function () {
+    this.truncated;
+    this.time;
+    this.category;
+    this.tags = new MapValue();
+    this.fields = new MapValue();
+    this.content;
+}
+
+module.exports = LineLog;

package/lib/logsink/log-sender.js

@@ -0,0 +1,78 @@
+var Configure = require('../conf/configure');
+var RequestQueue = require('../util/request-queue');
+var Logger = require('../logger');
+var SenderUtil = require('./sender-util');
+var conf = require("../conf/configure");
+
+var logsink_enabled = Configure.getProperty('logsink_enabled', false);
+var logsink_queue_size = Configure.getProperty('logsink_queue_size', 1000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled) {
+        LogSender.getInstance().startProcessQueue();
+    } else {
+        LogSender.getInstance().stopProcessQueue();
+        LogSender.resetInstance();
+    }
+});
+
+class LogSender {
+    constructor() {
+        if (LogSender.instance) {
+            return LogSender.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.isProcessing = false;
+        LogSender.instance = this;
+
+        if (logsink_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!LogSender.instance) {
+            LogSender.instance = new LogSender();
+        }
+        return LogSender.instance;
+    }
+
+    static resetInstance() {
+        if(LogSender.instance){
+            LogSender.instance = null;
+        }
+    }
+
+    add(log) {
+        this.queue.put(log);
+    }
+
+    async processQueue() {
+        this.isProcessing = true;
+
+        while (this.isProcessing) {
+            var log = await this.queue.get();
+            if(log){
+                try {
+                    var sendUtil = new SenderUtil();
+                    sendUtil.send(log);
+                } catch (error) {
+                    Logger.printError('WHATAP-702', 'Error occurred during send LogSinkPack.' + error, false);
+                }
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue();
+        }
+    }
+
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = LogSender;

package/lib/logsink/log-tracer.js

@@ -0,0 +1,40 @@
+var conf = require('../conf/configure');
+var LineLog = require('./line-log');
+var LineLogUtil = require('./line-log-util');
+var LogSender = require('./log-sender');
+var TraceContextManager = require('../trace/trace-context-manager');
+
+var LogTracer = function () {
+    this.logSender = LogSender.getInstance();
+}
+
+LogTracer.prototype.addStdWrite = function (content, category) {
+    var lineLog = new LineLog();
+    lineLog.category = category
+
+    var lineLogUtil = LineLogUtil.getInstance();
+    lineLogUtil.checkLogContent(lineLog, content);
+
+    if(lineLog.content){
+        this.addTxTag(lineLog);
+        this.logSender.add(lineLog)
+    }
+}
+
+LogTracer.prototype.addTxTag = function (lineLog) {
+    if(!lineLog){
+        return;
+    }
+
+    var ctx = TraceContextManager.getCurrentContext();
+    if(ctx){
+        if(conf.getProperty('logsink_trace_txid_enabled', true) && !ctx.txid.isZero()){
+            lineLog.fields.putLong('@txid', ctx.txid);
+        }
+        if(conf.getProperty('logsink_trace_mtid_enabled', true) && !ctx.mtid.isZero()){
+            lineLog.fields.putLong('@mtid', ctx.mtid);
+        }
+    }
+}
+
+module.exports = LogTracer;

package/lib/logsink/sender-util.js

@@ -0,0 +1,51 @@
+var conf = require('../conf/configure');
+var SecurityMaster      = require('../net/security-master');
+var LogSinkPack = require('../pack/log-sink-pack');
+var DataPackSender = require('../data/datapack-sender');
+var DateUtil = require('../util/dateutil')
+var ZipSend = require('./zip/zip-send');
+
+function SenderUtil() {
+    this.conf = conf;
+    this.secu = SecurityMaster;
+}
+
+SenderUtil.prototype.send = function (log) {
+    const p = new LogSinkPack();
+    p.time = log.time;
+    if (!p.time) {
+        p.time = DateUtil.currentTime();
+    }
+    p.category = log.category;
+
+    if (this.secu.ONAME) {
+        p.tags.putString('oname', this.secu.ONAME);
+    }
+    if (this.conf.OKIND) {
+        p.tags.putString('okindName', this.conf.OKIND_NAME);
+    }
+    if (this.conf.ONODE) {
+        p.tags.putString('onodeName', this.conf.ONODE_NAME);
+    }
+
+    if (log.tags != null) {
+        p.tags.putAllMapValue(log.tags);
+    }
+    if (log.fields != null) {
+        if (p.fields == null) {
+            p.fields = log.fields;
+        } else {
+            p.fields.putAllMapValue(log.fields);
+        }
+    }
+
+    p.content = log.content;
+
+    if(conf.getProperty('logsink_zip_enabled', false)){
+        ZipSend.getInstance().add(p);
+    }else{
+        DataPackSender.sendLogSinkPack(p)
+    }
+};
+
+module.exports = SenderUtil;

package/lib/logsink/zip/zip-send.js

@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../../conf/configure'),
+    DateUtil        = require('../../util/dateutil'),
+    RequestQueue    = require('../../util/request-queue'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataOutputX = require('../../io/data-outputx'),
+    ZipPack = require('../../pack/zip-pack'),
+    Logger          = require('../../logger');
+var zlib = require('zlib');
+
+var logsink_enabled = conf.getProperty('logsink_enabled', false);
+var logsink_zip_enabled = conf.getProperty('logsink_zip_enabled', false);
+var logsink_max_buffer_size = conf.getProperty('logsink_max_buffer_size', 1024*64);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var logsink_queue_size = conf.getProperty('logsink_queue_size', 500);
+var logsink_max_wait_time = conf.getProperty('logsink_max_wait_time', 2000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+conf.on('logsink_zip_enabled', function(newProperty) {
+    logsink_zip_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+
+class ZipSend {
+    constructor() {
+        if (ZipSend.instance) {
+            return ZipSend.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipSend.instance = this;
+
+        if (logsink_enabled && logsink_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipSend.instance) {
+            ZipSend.instance = new ZipSend();
+        }
+        return ZipSend.instance;
+    }
+
+    static resetInstance() {
+        if(ZipSend.instance){
+            ZipSend.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendLogSinkPack(p);
+        }
+    }
+
+    // async addWait(p, waitTimeForFull) {
+    //     var ok = this.queue.put(p);
+    //     if (ok === false){
+    //         if(waitTimeForFull > 0){
+    //             while(this.queue.put(p) === false){
+    //                 await new Promise(resolve => setTimeout(resolve, waitTimeForFull));
+    //             }
+    //         }
+    //     }
+    // }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!logsink_enabled || !logsink_zip_enabled) {
+                this.isProcessing = false;
+                break;
+            }
+
+            const p = await this.queue.getByTimeout(logsink_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        this.packCount++;
+        this.buffer = Buffer.concat([this.buffer, b]);
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if (this.buffer.length >= logsink_max_buffer_size) {
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= logsink_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('logsink_max_wait_time', 2000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(conf.getProperty('debug_logsink_zip_enabled', false)){
+            Logger.print('WHATAP-703', `zip status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length}`);
+        }
+        DataPackSender.sendLogSinkPack(p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (error) {
+            Logger.print('WHATAP-701', 'Error occurred during compression.' + error, false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = ZipSend;

package/lib/net/paramdef.js

@@ -34,6 +34,7 @@ exports.AGENT_LOG_READ = 23;
 
 exports.GET_ACTIVE_TRANSACTION_LIST = 25;
 exports.GET_ACTIVE_TRANSACTION_DETAIL = 26;
+exports.GET_TOPOLOGY = 27
 	
 exports.NODE_MODULE_DEPENDENCY = 101;
 

package/lib/net/security-master.js

@@ -90,9 +90,11 @@ var SecurityMaster = {
             this.OKIND_NAME=null;
         }
 
-        if(Configuration.getProperty('whatap_micro_enabled', false) === false || process.env.WHATAP_MICRO_ENABLED === "true") {
-            var onodeStr= process.env.WHATAP_ONODE ||  Configuration.getProperty('whatap.onode');
-            console.log(onodeStr)
+        const whatapMicroEnabledConfig = Configuration.getProperty('whatap_micro_enabled', false);
+        const whatapMicroEnabledEnv = process.env.WHATAP_MICRO_ENABLED;
+
+        if (!whatapMicroEnabledConfig || whatapMicroEnabledEnv === undefined || whatapMicroEnabledEnv === "false") {
+            var onodeStr= process.env.WHATAP_ONODE || process.env.NODE_NAME ||  Configuration.getProperty('whatap.onode', null);
             if(onodeStr && onodeStr.length>0){
                 this.ONODE = HashUtil.hashFromString(onodeStr);
                 this.ONODE_NAME=onodeStr;
@@ -140,8 +142,11 @@ var SecurityMaster = {
             this.OKIND_NAME=null;
         }
 
-        if(Configuration.getProperty('whatap_micro_enabled', false) === false || process.env.WHATAP_MICRO_ENABLED === "true") {
-            var onodeStr= process.env.WHATAP_ONODE ||  Configuration.getProperty('whatap.onode');
+        const whatapMicroEnabledConfig = Configuration.getProperty('whatap_micro_enabled', false);
+        const whatapMicroEnabledEnv = process.env.WHATAP_MICRO_ENABLED;
+
+        if (!whatapMicroEnabledConfig || whatapMicroEnabledEnv === undefined || whatapMicroEnabledEnv === "false") {
+            var onodeStr= process.env.WHATAP_ONODE || process.env.NODE_NAME ||  Configuration.getProperty('whatap.onode', null);
             if(onodeStr && onodeStr.length>0){
                 this.ONODE = HashUtil.hashFromString(onodeStr);
                 this.ONODE_NAME=onodeStr;

package/lib/observers/global-observer.js

@@ -7,6 +7,15 @@
 var TraceContextManager = require('../trace/trace-context-manager'),
     Logger = require('../logger');
 
+var HttpStepX = require('../step/http-stepx');
+const Hexa32 = require("../util/hexa32");
+const SecurityMaster = require("../net/security-master");
+const conf = require("../conf/configure");
+const KeyGen = require("../util/keygen");
+const HashUtil = require("../util/hashutil");
+const DataTextAgent = require("../data/datatext-agent");
+const shimmer = require('../core/shimmer');
+
 var GlobalObserver = function (agent) {
     this.agent = agent;
     this.packages = ['global'];
@@ -26,6 +35,137 @@ GlobalObserver.prototype.inject = function (mod, moduleName) {
             });
         });
     });
+
+    shimmer.wrap(mod, 'fetch', function(original) {
+        return function(...args) {
+            var info = args[1] ? args[1] : {};
+            var ctx = TraceContextManager._asyncLocalStorage.getStore();
+
+            if (ctx) {
+                interTxTraceAutoOn(ctx);
+
+                if (conf.mtrace_enabled) {
+                    if (info.headers) {
+                        info.headers['x-wtap-po'] = transferPOID(ctx);
+                    } else {
+                        info.headers = {
+                            'x-wtap-po': transferPOID(ctx)
+                        };
+                    }
+                    if (conf.stat_mtrace_enabled) {
+                        info.headers[conf._trace_mtrace_spec_key1] = transferSPEC_URL(ctx);
+                    }
+                    if (conf.mtid_mtrace_enabled && ctx.mtid.isZero() === false) {
+                        info.headers[conf._trace_mtrace_caller_key] = transferMTID_CALLERTX(ctx);
+                    }
+                    args[1] = info;
+                }
+            }
+
+            const promise = original.apply(this, args);
+
+            promise.then(response => {
+                var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                if (ctx && args[0]) {
+                    const url = new URL(args[0]);
+                    ctx.httpc_host = url.hostname;
+                    ctx.httpc_url = url.pathname;
+                    ctx.httpc_port = url.port || (url.protocol === 'https:' ? 443 : 80);
+
+                    var step = new HttpStepX();
+                    step.start_time = ctx.getElapsedTime();
+                    step.url = HashUtil.hashFromString(ctx.httpc_url);
+                    DataTextAgent.HTTPC_URL.add(step.url, ctx.httpc_url);
+                    step.host = HashUtil.hashFromString(ctx.httpc_host);
+                    DataTextAgent.HTTPC_HOST.add(step.host, ctx.httpc_host);
+                    step.port = ctx.httpc_port;
+
+                    ctx.profile.push(step);
+                }
+            }).catch(e => {
+                Logger.printError("WHATAP-704", "Promise error occurred during fetch: " + e, false);
+            });
+            return promise;
+        };
+    });
 };
 
+var transfer_poid;
+function transferPOID(ctx) {
+    if (transfer_poid)
+        return transfer_poid;
+    transfer_poid = Hexa32.toString32(SecurityMaster.PCODE) + ','
+        + Hexa32.toString32(SecurityMaster.OKIND) + ',' + Hexa32.toString32(SecurityMaster.OID);
+    return transfer_poid;
+}
+
+function transferMTID_CALLERTX(ctx) {
+    if (ctx.transfer_id)
+        return ctx.transfer_id;
+    var x = Hexa32.toString32(ctx.mtid) + ',' + (ctx.mdepth + 1) + ',' + Hexa32.toString32(ctx.txid);
+    ctx.transfer_id = x;
+    return ctx.transfer_id;
+}
+
+function transferSPEC_URL(ctx) {
+    if (ctx.transfer_info)
+        return ctx.transfer_info;
+    var x = conf.mtrace_spec + ',' + ctx.service_hash;
+    ctx.transfer_info = x;
+    return ctx.transfer_info;
+}
+
+var check_seq = 1;
+function interTxTraceAutoOn(ctx) {
+    if (conf.mtrace_enabled == false || ctx.httpc_checked || ctx.mtid.isZero() === false)
+        return;
+    ctx.httpc_checked = true;
+    if (conf.mtrace_rate >= 100) {
+        ctx.mtid = KeyGen.next();
+        return;
+    }
+    check_seq++;
+    switch (Math.floor(conf.mtrace_rate / 10)) {
+        case 10:
+            ctx.mtid = KeyGen.next();
+            break;
+        case 9:
+            if (check_seq % 10 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 8:
+            if (check_seq % 5 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 7:
+            if (check_seq % 4 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 6:
+            if (check_seq % 3 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 5:
+            if (check_seq % 2 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 4:
+            if (check_seq % 3 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 3:
+            if (check_seq % 4 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 2:
+            if (check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 1:
+            if (check_seq % 10 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+    }
+}
+
 exports.GlobalObserver = GlobalObserver;